From d878c81ce84b9c2e750e8aa7cdb07c795f2eef2a Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Sun, 14 Jun 2026 18:53:18 -0500 Subject: [PATCH 01/86] Clamp position precision on public / known-keys (#10665) * Clamp position precision on public / known-keys (Compliance) * Fix review comments: bounds check, all-channel precision clamp, disabled channel guard * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Refactor position precision comments for clarity and update channel configuration handling in AdminModule * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/mesh/Channels.cpp | 36 ++++++++ src/mesh/Channels.h | 6 ++ src/mesh/PositionPrecision.cpp | 11 ++- src/mesh/PositionPrecision.h | 9 ++ src/modules/AdminModule.cpp | 20 +++++ src/modules/AdminModule.h | 3 + test/test_position_precision/test_main.cpp | 96 ++++++++++++++++++++++ 7 files changed, 180 insertions(+), 1 deletion(-) diff --git a/src/mesh/Channels.cpp b/src/mesh/Channels.cpp index be75e3d42..457e64461 100644 --- a/src/mesh/Channels.cpp +++ b/src/mesh/Channels.cpp @@ -404,6 +404,42 @@ bool Channels::isDefaultChannel(ChannelIndex chIndex) return false; } +bool cryptoKeyIsPublic(const CryptoKey &key) +{ + if (key.length == 0) + return true; // encryption disabled + // Match the defaultpsk family ignoring its last byte (getKey() bumps only that byte per 1-byte index). + if (key.length == (int)sizeof(defaultpsk) && memcmp(key.bytes, defaultpsk, sizeof(defaultpsk) - 1) == 0) + return true; + return false; +} + +bool Channels::usesPublicKey(ChannelIndex chIndex) +{ + const meshtastic_Channel &ch = getByIndex(chIndex); + if (!ch.has_settings || ch.role == meshtastic_Channel_Role_DISABLED) + return false; + + const auto &psk = ch.settings.psk; + if (psk.size == 0) { + // Secondary channels inherit the primary key when unset; primary size==0 means encryption disabled. + if (ch.role == meshtastic_Channel_Role_SECONDARY) { + // Guard against malformed configs with no PRIMARY channel (primaryIndex could point back to us). + if (primaryIndex == chIndex) + return true; // fail closed: treat as public + return usesPublicKey(primaryIndex); + } + return true; + } + + if (psk.size == 1) { + // Short PSK aliases: 0 disables encryption; 1..255 are the public defaultpsk family. + return true; + } + + return (psk.size == sizeof(defaultpsk) && memcmp(psk.bytes, defaultpsk, sizeof(defaultpsk) - 1) == 0); +} + bool Channels::hasDefaultChannel() { // If we don't use a preset or the default frequency slot, or we override the frequency, we don't have a default channel diff --git a/src/mesh/Channels.h b/src/mesh/Channels.h index a3cc7791c..9ba84e2ee 100644 --- a/src/mesh/Channels.h +++ b/src/mesh/Channels.h @@ -86,6 +86,9 @@ class Channels // Returns true if the channel has the default name and PSK bool isDefaultChannel(ChannelIndex chIndex); + // Returns true if this channel's effective key is publicly decryptable (open or well-known/default PSK). + bool usesPublicKey(ChannelIndex chIndex); + // Returns true if we can be reached via a channel with the default settings given a region and modem preset bool hasDefaultChannel(); @@ -144,6 +147,9 @@ extern Channels channels; static const uint8_t defaultpsk[] = {0xd4, 0xf1, 0xbb, 0x3a, 0x20, 0x29, 0x07, 0x59, 0xf0, 0xbc, 0xff, 0xab, 0xcf, 0x4e, 0x69, 0x01}; +/// True if a getKey()-resolved key offers no privacy: length 0 (off) or the public defaultpsk family. Pure; for tests. +bool cryptoKeyIsPublic(const CryptoKey &key); + static const uint8_t eventpsk[] = {0x38, 0x4b, 0xbc, 0xc0, 0x1d, 0xc0, 0x22, 0xd1, 0x81, 0xbf, 0x36, 0xb8, 0x61, 0x21, 0xe1, 0xfb, 0x96, 0xb7, 0x2e, 0x55, 0xbf, 0x74, 0x22, 0x7e, 0x9d, 0x6a, 0xfb, 0x48, 0xd6, 0x4c, 0xb1, 0xa1}; \ No newline at end of file diff --git a/src/mesh/PositionPrecision.cpp b/src/mesh/PositionPrecision.cpp index 75a17d6e9..b5b29903b 100644 --- a/src/mesh/PositionPrecision.cpp +++ b/src/mesh/PositionPrecision.cpp @@ -16,7 +16,16 @@ uint32_t getPositionPrecisionForChannel(const meshtastic_Channel &channel) uint32_t getPositionPrecisionForChannel(uint8_t channelIndex) { - return getPositionPrecisionForChannel(channels.getByIndex(channelIndex)); + const meshtastic_Channel &ch = channels.getByIndex(channelIndex); + if (ch.role == meshtastic_Channel_Role_DISABLED) + return 0; + uint32_t precision = getPositionPrecisionForChannel(ch); + + // Never send a precise position on a publicly-decryptable channel (key check is gated on > ceiling). + if (precision > MAX_POSITION_PRECISION_PUBLIC_KEY && channels.usesPublicKey(channelIndex)) { + precision = MAX_POSITION_PRECISION_PUBLIC_KEY; + } + return precision; } static int32_t truncateCoordinate(int32_t coordinate, uint32_t precision) diff --git a/src/mesh/PositionPrecision.h b/src/mesh/PositionPrecision.h index 89828f2e0..1955cf80f 100644 --- a/src/mesh/PositionPrecision.h +++ b/src/mesh/PositionPrecision.h @@ -4,7 +4,16 @@ #include "meshtastic/mesh.pb.h" #include +// Max precision on a publicly-decryptable channel. CCPA "precise geolocation" = within a ~564m (1,850ft) radius. +// Precision is bit-truncation of latitude_i/longitude_i: the latitude cell stays ~constant in meters worldwide +// (~700m at 15 bits), while only the longitude cell varies — widest at the equator, narrowing toward the poles. +// 15 also matches the MQTT map-report public precision ceiling. +#define MAX_POSITION_PRECISION_PUBLIC_KEY 15 + +// Configured precision as-is; does NOT apply the public-key clamp -- use the channelIndex overload for the on-wire value. uint32_t getPositionPrecisionForChannel(const meshtastic_Channel &channel); + +// Configured precision, clamped to MAX_POSITION_PRECISION_PUBLIC_KEY when the channel's effective key is publicly decryptable. uint32_t getPositionPrecisionForChannel(uint8_t channelIndex); void applyPositionPrecision(meshtastic_Position &position, uint32_t precision); bool applyPositionPrecision(meshtastic_MeshPacket &packet, uint32_t precision); diff --git a/src/modules/AdminModule.cpp b/src/modules/AdminModule.cpp index d79261d5d..8715e202d 100644 --- a/src/modules/AdminModule.cpp +++ b/src/modules/AdminModule.cpp @@ -2,6 +2,7 @@ #include "Channels.h" #include "MeshService.h" #include "NodeDB.h" +#include "PositionPrecision.h" #include "PowerFSM.h" #include "RTC.h" #include "SPILock.h" @@ -1157,7 +1158,26 @@ void AdminModule::handleSetChannel(const meshtastic_Channel &cc) if (channels.ensureLicensedOperation()) { sendWarning(licensedModeMessage); } + // Refresh derived state (primaryIndex in particular) BEFORE the precision clamp below. usesPublicKey() + // resolves a secondary channel's key against the primary, so it must see the post-update primaryIndex; + // running the clamp first could evaluate secondaries against the previous primary and skip the clamp/warning. channels.onConfigChanged(); // tell the radios about this change + + // Persist the public-key precision clamp for all channels that may be affected (e.g. secondaries + // that inherit a now-public primary key) and warn the client once if anything was coarsened. + bool clamped = false; + for (uint8_t i = 0; i < channels.getNumChannels(); i++) { + meshtastic_Channel &ch = channels.getByIndex(i); + if (ch.role == meshtastic_Channel_Role_DISABLED || !ch.settings.has_module_settings) + continue; + uint32_t allowed = getPositionPrecisionForChannel(i); + if (allowed != ch.settings.module_settings.position_precision) { + ch.settings.module_settings.position_precision = allowed; + clamped = true; + } + } + if (clamped) + sendWarning(publicChannelPrecisionMessage); saveChanges(SEGMENT_CHANNELS, false); } diff --git a/src/modules/AdminModule.h b/src/modules/AdminModule.h index aeb41471c..1d5c64423 100644 --- a/src/modules/AdminModule.h +++ b/src/modules/AdminModule.h @@ -88,6 +88,9 @@ class AdminModule : public ProtobufModule, public Obser static constexpr const char *licensedModeMessage = "Licensed mode activated, removing admin channel and encryption from all channels"; +static constexpr const char *publicChannelPrecisionMessage = + "Precise position is not allowed on a public (open / known-key) channel; reduced to coarse precision"; + extern AdminModule *adminModule; void disableBluetooth(); \ No newline at end of file diff --git a/test/test_position_precision/test_main.cpp b/test/test_position_precision/test_main.cpp index bb6118177..f7727e0d0 100644 --- a/test/test_position_precision/test_main.cpp +++ b/test/test_position_precision/test_main.cpp @@ -1,6 +1,8 @@ +#include "Channels.h" #include "PositionPrecision.h" #include "TestUtil.h" #include "mesh-pb-constants.h" +#include #include static meshtastic_Position makePosition() @@ -119,6 +121,92 @@ static void test_getPositionPrecisionForChannel_secondaryWithoutModuleSettingsFa TEST_ASSERT_EQUAL_UINT32(0, getPositionPrecisionForChannel(channel)); } +// End-to-end via the channelIndex overload + live channels singleton, exercising getKey()'s 1-byte->16-byte expansion. +static void test_getPositionPrecisionForChannel_clampsPreciseOnDefaultKeyChannel() +{ + channels.initDefaults(); // channel 0: primary, default key (psk {0x01}) -> publicly decryptable + uint8_t idx = 0; + meshtastic_Channel &ch = channels.getByIndex(idx); + ch.settings.has_module_settings = true; + ch.settings.module_settings.position_precision = 32; // user requests "Precise" on a public channel + + TEST_ASSERT_EQUAL_UINT32(MAX_POSITION_PRECISION_PUBLIC_KEY, getPositionPrecisionForChannel(idx)); +} + +static void test_getPositionPrecisionForChannel_keepsPreciseOnStrongKeyChannel() +{ + channels.initDefaults(); + uint8_t idx = 0; + meshtastic_Channel &ch = channels.getByIndex(idx); + memset(ch.settings.psk.bytes, 0xAB, 16); // a private 128-bit key, not the defaultpsk family + ch.settings.psk.size = 16; + ch.settings.has_module_settings = true; + ch.settings.module_settings.position_precision = 32; + + TEST_ASSERT_EQUAL_UINT32(32, getPositionPrecisionForChannel(idx)); +} + +static CryptoKey makeCryptoKey(const uint8_t *bytes, int length) +{ + CryptoKey k; + memset(k.bytes, 0, sizeof(k.bytes)); + + // CryptoKey::length is int8_t and CryptoKey::bytes is 32 bytes; keep the helper consistent and overflow-safe. + int cappedLen = length; + if (cappedLen < 0) + cappedLen = -1; + else if (cappedLen > static_cast(sizeof(k.bytes))) + cappedLen = static_cast(sizeof(k.bytes)); + + if (cappedLen > 0 && bytes != nullptr) { + memcpy(k.bytes, bytes, static_cast(cappedLen)); + } + + k.length = static_cast(cappedLen); + return k; +} + +static void test_cryptoKeyIsPublic_openKeyIsPublic() +{ + // length 0 == encryption disabled. + TEST_ASSERT_TRUE(cryptoKeyIsPublic(makeCryptoKey(nullptr, 0))); +} + +static void test_cryptoKeyIsPublic_defaultKeyIsPublic() +{ + // The expanded default PSK (the 16-byte defaultpsk) -- the case a key-length check misses. + TEST_ASSERT_TRUE(cryptoKeyIsPublic(makeCryptoKey(defaultpsk, sizeof(defaultpsk)))); +} + +static void test_cryptoKeyIsPublic_defaultKeyFamilyVariesLastByte() +{ + // Higher indices (e.g. {0x02}) expand to defaultpsk with only the last byte bumped -- still public. + uint8_t key[sizeof(defaultpsk)]; + memcpy(key, defaultpsk, sizeof(defaultpsk)); + key[sizeof(defaultpsk) - 1] = static_cast(key[sizeof(defaultpsk) - 1] + 1); + TEST_ASSERT_TRUE(cryptoKeyIsPublic(makeCryptoKey(key, sizeof(key)))); +} + +static void test_cryptoKeyIsPublic_strongKeyIsPrivate() +{ + uint8_t key[16]; + memset(key, 0xAB, sizeof(key)); // not the defaultpsk family + TEST_ASSERT_FALSE(cryptoKeyIsPublic(makeCryptoKey(key, sizeof(key)))); +} + +static void test_cryptoKeyIsPublic_aes256KeyIsPrivate() +{ + uint8_t key[32]; + memset(key, 0x11, sizeof(key)); + TEST_ASSERT_FALSE(cryptoKeyIsPublic(makeCryptoKey(key, sizeof(key)))); +} + +static void test_cryptoKeyIsPublic_invalidKeyIsNotPublic() +{ + // length < 0 == no/invalid key (e.g. a disabled channel); it carries no traffic to leak. + TEST_ASSERT_FALSE(cryptoKeyIsPublic(makeCryptoKey(nullptr, -1))); +} + void setUp(void) {} void tearDown(void) {} @@ -136,6 +224,14 @@ void setup() RUN_TEST(test_getPositionPrecisionForChannel_explicitZeroDisablesPrimary); RUN_TEST(test_getPositionPrecisionForChannel_primaryWithoutModuleSettingsFailsClosed); RUN_TEST(test_getPositionPrecisionForChannel_secondaryWithoutModuleSettingsFailsClosed); + RUN_TEST(test_getPositionPrecisionForChannel_clampsPreciseOnDefaultKeyChannel); + RUN_TEST(test_getPositionPrecisionForChannel_keepsPreciseOnStrongKeyChannel); + RUN_TEST(test_cryptoKeyIsPublic_openKeyIsPublic); + RUN_TEST(test_cryptoKeyIsPublic_defaultKeyIsPublic); + RUN_TEST(test_cryptoKeyIsPublic_defaultKeyFamilyVariesLastByte); + RUN_TEST(test_cryptoKeyIsPublic_strongKeyIsPrivate); + RUN_TEST(test_cryptoKeyIsPublic_aes256KeyIsPrivate); + RUN_TEST(test_cryptoKeyIsPublic_invalidKeyIsNotPublic); exit(UNITY_END()); } From 43d485dd76037c6f7d403de14c68ce7dad28b1b6 Mon Sep 17 00:00:00 2001 From: Jonathan Bennett Date: Mon, 15 Jun 2026 13:39:57 -0500 Subject: [PATCH 02/86] Add IIS2MDCTR and ISM330DHCX to ScanI2C (#10723) * add ScanI2C for IIS2MDCTR and ISM330DHCX * Trunk format * Minor cleanup Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/detect/ScanI2C.cpp | 10 +++++----- src/detect/ScanI2C.h | 2 ++ src/detect/ScanI2CTwoWire.cpp | 15 ++++++++++++++- variants/nrf52840/heltec_mesh_node_t1/variant.cpp | 3 +-- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/detect/ScanI2C.cpp b/src/detect/ScanI2C.cpp index aa25ab5e1..a147194a7 100644 --- a/src/detect/ScanI2C.cpp +++ b/src/detect/ScanI2C.cpp @@ -37,15 +37,15 @@ ScanI2C::FoundDevice ScanI2C::firstKeyboard() const ScanI2C::FoundDevice ScanI2C::firstAccelerometer() const { - ScanI2C::DeviceType types[] = {MPU6050, LIS3DH, BMA423, LSM6DS3, BMX160, STK8BAXX, - ICM20948, QMA6100P, BMM150, BMI270, ICM42607P}; - return firstOfOrNONE(11, types); + ScanI2C::DeviceType types[] = {MPU6050, LIS3DH, BMA423, LSM6DS3, BMX160, STK8BAXX, + ICM20948, QMA6100P, BMM150, BMI270, ICM42607P, ISM330DHCX}; + return firstOfOrNONE(12, types); } ScanI2C::FoundDevice ScanI2C::firstMagnetometer() const { - ScanI2C::DeviceType types[] = {MMC5983MA}; - return firstOfOrNONE(1, types); + ScanI2C::DeviceType types[] = {MMC5983MA, IIS2MDCTR}; + return firstOfOrNONE(2, types); } ScanI2C::FoundDevice ScanI2C::firstAQI() const diff --git a/src/detect/ScanI2C.h b/src/detect/ScanI2C.h index 7c4f2f0e3..2333601f7 100644 --- a/src/detect/ScanI2C.h +++ b/src/detect/ScanI2C.h @@ -99,6 +99,8 @@ class ScanI2C CW2015, SCD30, ADS1115, + IIS2MDCTR, + ISM330DHCX, } DeviceType; // typedef uint8_t DeviceAddress; diff --git a/src/detect/ScanI2CTwoWire.cpp b/src/detect/ScanI2CTwoWire.cpp index 5a9ec9a65..b8b6915c2 100644 --- a/src/detect/ScanI2CTwoWire.cpp +++ b/src/detect/ScanI2CTwoWire.cpp @@ -584,6 +584,9 @@ void ScanI2CTwoWire::scanPort(I2CPort port, uint8_t *address, uint8_t asize) if (registerValue == 0x6A) { type = LSM6DS3; logFoundDevice("LSM6DS3", (uint8_t)addr.address); + } else if (registerValue == 0x6B) { + type = ISM330DHCX; + logFoundDevice("ISM330DHCX", (uint8_t)addr.address); } else { type = QMI8658; logFoundDevice("QMI8658", (uint8_t)addr.address); @@ -591,7 +594,17 @@ void ScanI2CTwoWire::scanPort(I2CPort port, uint8_t *address, uint8_t asize) break; SCAN_SIMPLE_CASE(QMC5883L_ADDR, QMC5883L, "QMC5883L", (uint8_t)addr.address) - SCAN_SIMPLE_CASE(HMC5883L_ADDR, HMC5883L, "HMC5883L", (uint8_t)addr.address) + case HMC5883L_ADDR: + registerValue = getRegisterValue(ScanI2CTwoWire::RegisterLocation(addr, 0x4FU), 1); // get ID + if (registerValue == 0x40) { + type = IIS2MDCTR; + logFoundDevice("IIS2MDCTR", (uint8_t)addr.address); + break; + } else { + type = HMC5883L; + logFoundDevice("HMC5883L", (uint8_t)addr.address); + break; + } #ifdef HAS_QMA6100P SCAN_SIMPLE_CASE(QMA6100P_ADDR, QMA6100P, "QMA6100P", (uint8_t)addr.address) #else diff --git a/variants/nrf52840/heltec_mesh_node_t1/variant.cpp b/variants/nrf52840/heltec_mesh_node_t1/variant.cpp index eb719f17f..1ea214069 100644 --- a/variants/nrf52840/heltec_mesh_node_t1/variant.cpp +++ b/variants/nrf52840/heltec_mesh_node_t1/variant.cpp @@ -25,8 +25,7 @@ const uint32_t g_ADigitalPinMap[] = { // P0 - pins 0 and 1 are hardwired for xtal and should never be enabled - 0xff, 0xff, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, - 26, 27, 28, 29, 30, 31, + 0xff, 0xff, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, // P1 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47}; From e4f4d1f9e7b05749a0b581feddcfb1b7cbd94817 Mon Sep 17 00:00:00 2001 From: Tom <116762865+NomDeTom@users.noreply.github.com> Date: Mon, 15 Jun 2026 19:53:17 +0100 Subject: [PATCH 03/86] Ci test report filter (#10722) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci(test report): drop no-status testsuites from the PlatformIO report PlatformIO emits a self-closing row for every test_* dir x every hardware variant it can't run on native (~4900 rows). They carry no pass/fail/skip status and bury the suites that actually ran in the dorny Test Report. Strip them (in generate-reports, before the reporter) so the report shows only real results. The uploaded artifact keeps the full XML. Co-Authored-By: Claude Opus 4.8 (1M context) * Add bin/run-tests.sh: standardised local test verdict Self-contained RED/AMBER/GREEN runner: matches all pass/fail spellings and cross-checks the suite count against test/test_*/ so a missing suite shows AMBER. Co-Authored-By: Claude Opus 4.8 (1M context) * run-tests.sh: detect sanitizer faults + live build/test progress Two usability improvements to the local verdict script: 1. Sanitizer-fault detection. A sanitizer-instrumented (coverage) build aborts non-zero at EXIT on an ASan/LSan/UBSan/TSan fault — most often a LeakSanitizer leak — AFTER every test has printed [PASSED], so pio reports it as [ERRORED]/SIGHUP with no :FAIL: anywhere and it masquerades as a phantom failure. verdict_red now recognises the documented fault signatures (ERROR:/ WARNING: :, SUMMARY: :, Direct/Indirect leak of, heap-use-after-free, runtime error:, etc. — not the benign "failed to intercept" startup noise) and reports e.g. "RESULT: RED sanitizer fault — SUMMARY: AddressSanitizer: ...", plus the "run the binary bare (gdb hides it via ptrace)" recipe. Also flags the "all tests passed but aborted at exit" shape when the report was swallowed. 2. Progress trail. Long rebuilds were silent for minutes. A background heartbeat now appends a status line every few seconds to .pio/build//.runtests-progress (always — tail -f it to check a backgrounded/piped run) and live-updates the tty for interactive --quiet runs: build = objects recompiled / cached total + ETA; test = suites done / expected. The object-count baseline is cached per env in the gitignored build dir. Writes never touch the parsed verdict log; tty writes are guarded so a no-tty run emits no redirect-open error. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .github/workflows/test_native.yml | 8 + bin/run-tests.sh | 249 ++++++++++++++++++++++++++++++ 2 files changed, 257 insertions(+) create mode 100755 bin/run-tests.sh diff --git a/.github/workflows/test_native.yml b/.github/workflows/test_native.yml index a0af13ff3..9ff192a16 100644 --- a/.github/workflows/test_native.yml +++ b/.github/workflows/test_native.yml @@ -161,6 +161,14 @@ jobs: name: platformio-test-report-${{ steps.version.outputs.long }} merge-multiple: true + - name: Drop no-status testsuites from the report + # PlatformIO emits a self-closing row for every test_* dir + # crossed with every hardware variant it cannot run on the native host (~4900 rows). + # They carry no pass/fail/skip status and bury the suites that actually ran. Strip + # them so the Test Report lists only suites with a real status. Only the copy the + # reporter renders is trimmed; the uploaded artifact keeps the full XML. + run: sed -i -E 's#]*tests="0"[^>]*/>##g' testreport.xml + - name: Test Report uses: dorny/test-reporter@v3.0.0 with: diff --git a/bin/run-tests.sh b/bin/run-tests.sh new file mode 100755 index 000000000..5965999cd --- /dev/null +++ b/bin/run-tests.sh @@ -0,0 +1,249 @@ +#!/usr/bin/env bash +# Run native PlatformIO unit tests and emit a single, unambiguous RED/AMBER/GREEN verdict. +# +# Why this exists: PlatformIO reports failures three different ways ([FAILED], :FAIL:, +# [ERRORED]) and an all-pass run prints "N succeeded" with NO "0 failed" clause — so naive +# greps produce false greens (see .notes/test-passfail-filter.md). This script encodes the +# correct logic once, and cross-checks the number of suites that actually ran against the +# canonical set in test/ so a suite silently going missing shows up as AMBER, not green. +# +# Usage: +# ./bin/run-tests.sh # run all suites, full RAG + count cross-check +# ./bin/run-tests.sh -f test_utf8 # run one suite (no count cross-check) +# ./bin/run-tests.sh -e native # override env (default: coverage) +# ./bin/run-tests.sh --quiet # only print the final RESULT line +# +# Exit codes: 0 = GREEN, 1 = RED, 2 = AMBER. +# +# The final line is machine-readable, e.g.: +# RESULT: GREEN 19/19 suites passed +# RESULT: AMBER 17/19 suites ran (missing: test_radio test_serial) — all that ran passed +# RESULT: RED test_traffic_management: 1 failed (or: build/crash error) +# RESULT: RED sanitizer fault — SUMMARY: AddressSanitizer: 1272 byte(s) leaked (tests may have +# all passed; the coverage build aborts at exit on an ASan/LSan fault — often shown only +# as [ERRORED]/SIGHUP. The script names it and points at running the binary bare.) + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$ROOT_DIR" + +ENV="coverage" +FILTER="" +QUIET=false +PASSTHRU=() + +while [[ $# -gt 0 ]]; do + case "$1" in + -f) + FILTER="$2" + PASSTHRU+=("-f" "$2") + shift 2 + ;; + -e) + ENV="$2" + shift 2 + ;; + --quiet) + QUIET=true + shift + ;; + *) + PASSTHRU+=("$1") + shift + ;; + esac +done + +# Locate pio (PATH, then the standard PlatformIO venv). +PIO="$(command -v pio || command -v platformio || echo "$HOME/.platformio/penv/bin/pio")" +if [[ ! -x $PIO ]] && ! command -v "$PIO" >/dev/null 2>&1; then + echo "RESULT: RED pio not found (looked in PATH and ~/.platformio/penv/bin)" + exit 1 +fi + +LOG="$(mktemp -t meshtest.XXXXXX.log)" +MARKER="" +PROGRESS_PID="" +trap 'rm -f "$LOG" "${MARKER:-}"; [[ -n ${PROGRESS_PID:-} ]] && kill "$PROGRESS_PID" 2>/dev/null' EXIT + +# Canonical suite set = the directories in test/. This is the source of truth for +# "what should run"; a filtered run only expects its filtered suite. +mapfile -t ALL_SUITES < <(find test -maxdepth 1 -type d -name 'test_*' -printf '%f\n' | sort) +EXPECTED_COUNT=${#ALL_SUITES[@]} + +# Cached object-count for this env, written after each completed build (in the gitignored build +# dir). Used as the progress denominator: accurate for a full rebuild (every object recompiles), +# only a rough upper bound for an incremental run. +BASELINE_FILE=".pio/build/${ENV}/.runtests-objcount" + +# Progress trail file (gitignored build dir). ALWAYS written so a backgrounded/piped run can be +# checked mid-build with `tail -f` — that's the whole point: don't fly blind on a 20-min rebuild. +PROGRESS_FILE=".pio/build/${ENV}/.runtests-progress" + +# --- Progress heartbeat ------------------------------------------------------ +# Emit ONE status line every few seconds: build = objects (re)compiled this run / cached total + +# best-effort ETA; test = suites finished / expected. Appends to $PROGRESS_FILE always (tail it to +# check on a backgrounded run); also live-updates the tty when $5=1 (interactive --quiet). Never +# touches $LOG, which is parsed for the verdict, so piped/CI captures stay clean. +progress_monitor() { + local marker="$1" objtotal="$2" testtotal="$3" pfile="$4" totty="$5" start now el done ran eta line + start=$(date +%s) + while :; do + now=$(date +%s) + el=$((now - start)) + if grep -q 'Testing\.\.\.' "$LOG" 2>/dev/null; then + ran=$(grep -cE "${ENV}:test_[a-z0-9_]+ \[(PASSED|FAILED|ERRORED)\]" "$LOG" 2>/dev/null) + line=$(printf '[test] %s/%s suites done — %dm%02ds' "$ran" "$testtotal" $((el / 60)) $((el % 60))) + else + done=$(find ".pio/build/${ENV}" -name '*.o' -newer "$marker" 2>/dev/null | wc -l) + if ((objtotal > 0 && done > 0)); then + eta=$((objtotal > done ? (objtotal - done) * el / done : 0)) + line=$(printf '[build] %d/%d objs — %dm%02ds — ETA ~%dm%02ds' \ + "$done" "$objtotal" $((el / 60)) $((el % 60)) $((eta / 60)) $((eta % 60))) + else + # done==0 (incremental: nothing to rebuild yet) or no cached baseline — no ETA yet. + line=$(printf '[build] %d objs compiled — %dm%02ds' "$done" $((el / 60)) $((el % 60))) + fi + fi + printf '%s\n' "$line" >>"$pfile" 2>/dev/null # file trail (always) + [[ $totty == 1 ]] && printf '\r\033[K%s' "$line" >/dev/tty 2>/dev/null # live line (human) + sleep 4 + done +} + +# Launch the heartbeat for every run. It writes the progress file unconditionally; the live tty +# line only when interactive AND --quiet (where pio's own output is hidden — otherwise pio's +# streamed compile lines already show progress and a \r line would just fight them). +mkdir -p ".pio/build/${ENV}" 2>/dev/null || true +: >"$PROGRESS_FILE" 2>/dev/null || true +MARKER="$(mktemp -t meshtest-mark.XXXXXX)" +TOTTY=0 +{ $QUIET && [[ -t 1 ]]; } && TOTTY=1 +progress_monitor "$MARKER" "$(cat "$BASELINE_FILE" 2>/dev/null || echo 0)" \ + "$([[ -n $FILTER ]] && echo 1 || echo "$EXPECTED_COUNT")" "$PROGRESS_FILE" "$TOTTY" & +PROGRESS_PID=$! + +if ! $QUIET; then + echo "Running: $PIO test -e $ENV ${PASSTHRU[*]-} (expecting $EXPECTED_COUNT suites)" +fi +echo "progress: tail -f $PROGRESS_FILE" >&2 + +# Run pio, tee to log. PIPESTATUS[0] is pio's real exit (NOT tee's). +if $QUIET; then + "$PIO" test -e "$ENV" "${PASSTHRU[@]}" >"$LOG" 2>&1 +else + "$PIO" test -e "$ENV" "${PASSTHRU[@]}" 2>&1 | tee "$LOG" +fi +PIO_RC=${PIPESTATUS[0]} + +# Stop the heartbeat, clear its line, and cache this build's object total for next time. +if [[ -n $PROGRESS_PID ]]; then + kill "$PROGRESS_PID" 2>/dev/null + wait "$PROGRESS_PID" 2>/dev/null + PROGRESS_PID="" + # Clear the live line only if we were writing one — opening /dev/tty when there is none is + # itself a redirect-open error the trailing 2>/dev/null cannot suppress. + [[ $TOTTY == 1 ]] && printf '\r\033[K' >/dev/tty 2>/dev/null +fi +[[ -d ".pio/build/${ENV}" ]] && find ".pio/build/${ENV}" -name '*.o' 2>/dev/null | wc -l >"$BASELINE_FILE" 2>/dev/null || true + +# --- Outcome detection ------------------------------------------------------- +# The SAME outcome is spelled differently depending on which layer emitted the line — this is +# the trap that produces false greens (grepping ":PASS" misses pio's "[PASSED]", grepping +# "[FAILED]" misses Unity's ":FAIL:"). So every regex below matches BOTH spellings: +# pass: Unity per-assertion ":PASS" | pio per-suite "[PASSED]" | summary "N succeeded" +# fail: Unity per-assertion ":FAIL:" | pio per-suite "[FAILED]" | summary "M failed" +# error: pio build/crash "[ERRORED]" | Unity "M Failures" | compiler "error:" +# Match \b after :PASS/:FAIL so ":PASSED"/":FAILED" forms are also caught either way. +FAIL_RE=':FAIL\b|\[FAILED\]|\[ERRORED\]|[1-9][0-9]* failed|[0-9]+ Tests [1-9][0-9]* Failures|error:|undefined reference|Segmentation fault|terminate called|SIGHUP|SIGSEGV|SIGABRT' +# Positive proof tests actually ran & passed (absence != success). Accept any pass spelling: +# the per-test/per-suite tokens OR a success summary line. +PASS_RE=':PASS\b|\[PASSED\]|test cases: *[0-9]+ succeeded|[0-9]+ Tests 0 Failures' +# Sanitizer (ASan/LSan/UBSan/TSan) fault signatures. The coverage build is sanitizer-instrumented +# and aborts NON-ZERO at exit on a fault — most often a LeakSanitizer leak — AFTER every test has +# already printed [PASSED]. pio then reports [ERRORED]/SIGHUP with no :FAIL: anywhere, so it +# masquerades as a phantom "N-1 of N succeeded". See .notes/test-passfail-filter.md. +# Match only real FAULT lines, never the benign "AddressSanitizer: failed to intercept '...'" +# startup noise that prints on every sanitizer run (it'd mislabel a normal [FAILED] as a leak). +# Formats per LLVM/Google sanitizer docs: ASan/LSan emit "==PID==ERROR: : ...", UBSan emits +# "file:line:col: runtime error: ...", TSan emits "WARNING: ThreadSanitizer: ..."; all close with +# a "SUMMARY: : ..." line (LSan-under-ASan reports its SUMMARY as "AddressSanitizer"). +SAN_RE='(ERROR|WARNING): (Address|Leak|Thread|UndefinedBehavior)Sanitizer:|SUMMARY: (Address|Leak|Thread|UndefinedBehavior)Sanitizer:|Direct leak of|Indirect leak of|detected memory leaks|heap-use-after-free|heap-buffer-overflow|stack-buffer-overflow|attempting double-free|LeakSanitizer has encountered a fatal error|runtime error:' + +# Suites that produced a per-suite verdict. pio emits "coverage:test_x [PASSED|FAILED|ERRORED]"; +# a SKIPPED suite (hardware-only on native) is "accounted for" too, so it doesn't read as missing. +mapfile -t RAN_SUITES < <(grep -oE "${ENV}:test_[a-z0-9_]+ \[(PASSED|FAILED|ERRORED)\]" "$LOG" | + sed -E "s/^${ENV}:(test_[a-z0-9_]+) .*/\1/" | sort -u) +RAN_COUNT=${#RAN_SUITES[@]} +# Suites pio explicitly skipped (don't count these as "missing" in the canonical cross-check). +mapfile -t SKIPPED_SUITES < <(grep -oE "${ENV}:test_[a-z0-9_]+.*\bSKIPPED\b" "$LOG" | + grep -oE "test_[a-z0-9_]+" | sort -u) + +verdict_red() { + local detail bin + detail="$(grep -nE '\[FAILED\]|:FAIL:|\[ERRORED\]' "$LOG" | head -3 | sed 's/^/ /')" + echo "" + echo "RED — failures detected:" + [[ -n $detail ]] && echo "$detail" + grep -E 'test cases:' "$LOG" | tail -1 | sed 's/^/ /' + + # Path to the test binary for the "run it bare" hint. For native/coverage the test program is + # the env executable (e.g. .pio/build/coverage/meshtasticd), NOT a file named 'program'. + bin="$(find ".pio/build/${ENV}" -maxdepth 1 -type f -executable ! -name '*.so' 2>/dev/null | head -1)" + [[ -z $bin ]] && bin=".pio/build/${ENV}/ (build it first: $PIO test -e ${ENV} ${FILTER:+-f $FILTER} --without-testing)" + + # Sanitizer fault (ASan/LSan/UBSan/TSan): name the real cause instead of "build/crash error". + if grep -qE "$SAN_RE" "$LOG"; then + grep -nE "$SAN_RE" "$LOG" | head -4 | sed 's/^/ /' + echo " -> sanitizer fault: if every test above is PASS, this is an exit-time abort, not a failed assertion." + echo " -> read the full report by running the binary BARE (gdb hides it via ptrace): ./$bin 2>&1 | tail -40" + echo "RESULT: RED sanitizer fault — $(grep -ohE 'SUMMARY: [A-Za-z]+Sanitizer:.*' "$LOG" | tail -1 || echo 'see report above')" + exit 1 + fi + + # All tests passed but the process still aborted at EXIT (ERRORED/SIGHUP/SIGABRT) and the + # sanitizer report was swallowed by the runner (often surfaced only as SIGHUP). Almost always a + # sanitizer fault — point at how to surface it rather than calling it a generic crash. + if grep -qE "$PASS_RE" "$LOG" && grep -qE '\[ERRORED\]|SIGHUP|SIGABRT' "$LOG" && ! grep -qE ':FAIL\b|\[FAILED\]' "$LOG"; then + echo " -> all tests passed but the process aborted at EXIT — likely an ASan/LSan fault whose report" + echo " the runner swallowed (commonly shown as SIGHUP). Run the binary BARE to see it: ./$bin 2>&1 | tail -40" + echo "RESULT: RED exit-time abort (tests passed; likely sanitizer — see hint above)" + exit 1 + fi + + echo "RESULT: RED $(grep -oE '[0-9]+ failed' "$LOG" | tail -1 || echo 'build/crash error')" + exit 1 +} + +# RED: pio non-zero, any failure marker, or no positive summary at all (build died early). +if [[ $PIO_RC -ne 0 ]] || grep -qE "$FAIL_RE" "$LOG"; then + verdict_red +fi +if ! grep -qE "$PASS_RE" "$LOG"; then + echo "" + echo "RESULT: RED no success summary found (build error / no tests ran?) — see log" + exit 1 +fi + +# AMBER: everything that ran passed, but (full run only) a canonical suite neither ran NOR was +# explicitly skipped — i.e. it silently went missing. SKIPPED suites are accounted for. +ACCOUNTED_COUNT=$((RAN_COUNT + ${#SKIPPED_SUITES[@]})) +if [[ -z $FILTER && $ACCOUNTED_COUNT -lt $EXPECTED_COUNT ]]; then + missing=() + for s in "${ALL_SUITES[@]}"; do + printf '%s\n' "${RAN_SUITES[@]}" "${SKIPPED_SUITES[@]}" | grep -qx "$s" || missing+=("$s") + done + echo "" + echo "RESULT: AMBER ${RAN_COUNT}/${EXPECTED_COUNT} suites ran (missing: ${missing[*]}) — all that ran passed" + exit 2 +fi + +# GREEN. +if [[ -n $FILTER ]]; then + echo "RESULT: GREEN ${RAN_COUNT} suite(s) passed (filtered: $FILTER)" +else + echo "RESULT: GREEN ${RAN_COUNT}/${EXPECTED_COUNT} suites passed" +fi +exit 0 From a944e1c908bc5f6c118b9527db97150ef760854c Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Wed, 17 Jun 2026 05:43:24 -0500 Subject: [PATCH 04/86] Update security documentation with detailed cryptographic mechanisms and known limitations --- SECURITY.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/SECURITY.md b/SECURITY.md index a77a4d028..34977e15a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -10,3 +10,35 @@ ## Reporting a Vulnerability We support the private reporting of potential security vulnerabilities. Please go to the Security tab to file a report with a description of the potential vulnerability and reproduction scripts (preferred) or steps, and our developers will review. + +Before filing, please read the Security Model below. Behavior whose only precondition is local API access to a node, or possession of a channel's pre-shared key, is intended by design and is not considered a vulnerability. + +## Security Model + +Meshtastic is an off-grid mesh protocol that runs on constrained microcontrollers within a 256 byte LoRa packet limit. These constraints shape its security design and rule out the heavier schemes used by IP-based protocols. This section summarizes what the firmware protects, the assumptions it rests on, and its known limits. Fuller write-ups are in the documentation: + +- Encryption overview: https://meshtastic.org/docs/overview/encryption/ +- Technical reference: https://meshtastic.org/docs/development/reference/encryption-technical/ +- Known limitations and future work: https://meshtastic.org/docs/about/overview/encryption/limitations/ + +### Cryptographic mechanisms + +- Channels are encrypted with a pre-shared key (PSK) using AES256-CTR. Channel traffic is encrypted but not authenticated, so anyone holding the PSK can read channel messages and can send messages as any node on that channel. +- Direct messages and admin messages use public key cryptography (x25519 key exchange with AES-CCM), providing confidentiality, authentication, and integrity between nodes on 2.5.0 or newer that have exchanged keys. +- Admin sessions use short-lived session IDs to limit replay of control messages. + +### Local trust boundary + +A client connected to a node over Bluetooth, USB serial, WiFi, or Ethernet has full local API access. From that connection it can read decrypted traffic, send messages as the node, change configuration (subject to managed mode), and read the node's private key for backup. This is intended behavior. The firmware trusts the local link the same way a phone or laptop trusts a directly attached device, and anything within reach of that connection (a shared LAN, a USB cable to an untrusted host, a paired phone) should be treated as part of the node itself. + +### Node identity (Trust On First Use) + +There is no central authority to sign node keys. The first public key a node hears for a given node number is the one it binds to that node number, a Trust On First Use (TOFU) model that is a hard requirement of a decentralized mesh. Clients and firmware reduce the impact of this by keeping favorited nodes from rolling out of the node database and by flagging public-key changes in the client UI. + +Firmware 2.8.X adds XEdDSA packet signing to further secure node identity claims and the authenticity of subsequent messages. It reuses each node's existing x25519 key pair to produce signatures, so a receiver can verify that a packet came from the holder of the bound key. Once a node has been seen signing, unsigned packets claiming that identity can be rejected. + +### Known limitations + +- No perfect forward secrecy. Traffic captured today can be decrypted later if a key is compromised, for example through a lost node or a mishandled channel key. +- Channel messages are not authenticated, as noted above. Although as of 2.8, channel messages will be xedDSA signed as a means of verification that is non-breaking. +- Setting WiFi credentials, or performing any other local administration, on an ESP32 over an untrusted network exposes that traffic, including the credentials, to the network. Provision and administer nodes over a trusted channel instead: Bluetooth, USB serial, or remote admin over the mesh. There is no current roadmap item to secure local administration over untrusted WiFi, though it may be addressed in a future release. From 2291b672c40d4998ea104e481ad2341af6418ac5 Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Wed, 17 Jun 2026 06:14:40 -0500 Subject: [PATCH 05/86] Protos --- protobufs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protobufs b/protobufs index 1df6c1154..362516671 160000 --- a/protobufs +++ b/protobufs @@ -1 +1 @@ -Subproject commit 1df6c1154248c82abb21b73eb06c203f595780db +Subproject commit 36251667179496c1e1261f4e4e5b349a51e5e861 From 7424631a27c21c850213f267ad55a844d3a33f68 Mon Sep 17 00:00:00 2001 From: Jonathan Bennett Date: Tue, 16 Jun 2026 22:37:14 -0500 Subject: [PATCH 06/86] Beta fixes (#10728) * Wipe message Store on factory reset * Check for destination 0 in a new message, and convert to broadcast * Make sure CHARGE_LED_state gets turned off, to avoid stuck LEDs * Take the spiLock in MessageStore when clearing messages * Trunk * Add thinknode M5 voltage curve * Fix the oops --- src/MessageStore.cpp | 1 + src/mesh/NodeDB.cpp | 4 ++++ src/modules/CannedMessageModule.cpp | 12 ++++++++++-- src/modules/StatusLEDModule.cpp | 8 ++++++++ variants/esp32s3/ELECROW-ThinkNode-M5/variant.h | 4 +++- 5 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/MessageStore.cpp b/src/MessageStore.cpp index a7e6c5662..32e78f400 100644 --- a/src/MessageStore.cpp +++ b/src/MessageStore.cpp @@ -354,6 +354,7 @@ void MessageStore::clearAllMessages() resetMessagePool(); #ifdef FSCom + concurrency::LockGuard guard(spiLock); SafeFile f(filename.c_str(), false); uint8_t count = 0; f.write(&count, 1); // write "0 messages" diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index 2481eb8ca..01d554ab8 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -9,6 +9,7 @@ #include "FSCommon.h" #include "MeshRadio.h" #include "MeshService.h" +#include "MessageStore.h" #include "NodeDB.h" #include "PacketHistory.h" #include "PowerFSM.h" @@ -727,6 +728,9 @@ bool NodeDB::factoryReset(bool eraseBleBonds) if (transmitHistory) { transmitHistory->clear(); } +#if HAS_SCREEN + messageStore.clearAllMessages(); +#endif // second, install default state (this will deal with the duplicate mac address issue) installDefaultNodeDatabase(); installDefaultDeviceState(); diff --git a/src/modules/CannedMessageModule.cpp b/src/modules/CannedMessageModule.cpp index 1dbc3a668..5fde2b047 100644 --- a/src/modules/CannedMessageModule.cpp +++ b/src/modules/CannedMessageModule.cpp @@ -84,7 +84,11 @@ void CannedMessageModule::LaunchWithDestination(NodeNum newDest, uint8_t newChan // Do NOT override explicit broadcast replies // Only reuse lastDest in LaunchRepeatDestination() - dest = newDest; + if (newDest == 0) { + dest = NODENUM_BROADCAST; + } else { + dest = newDest; + } channel = newChannel; lastDest = dest; @@ -124,7 +128,11 @@ void CannedMessageModule::LaunchFreetextWithDestination(NodeNum newDest, uint8_t // Do NOT override explicit broadcast replies // Only reuse lastDest in LaunchRepeatDestination() - dest = newDest; + if (newDest == 0) { + dest = NODENUM_BROADCAST; + } else { + dest = newDest; + } channel = newChannel; lastDest = dest; diff --git a/src/modules/StatusLEDModule.cpp b/src/modules/StatusLEDModule.cpp index 72aed21ff..353317d25 100644 --- a/src/modules/StatusLEDModule.cpp +++ b/src/modules/StatusLEDModule.cpp @@ -131,6 +131,12 @@ int32_t StatusLEDModule::runOnce() CHARGE_LED_state = LED_STATE_OFF; } } + } else { +#if defined(LED_HEARTBEAT) + // If we are using the heartbeat, as in the Thinknode M4, we need to explicitly turn off the charge LED + // This probably implies that in the future we need to stop re-using this bool for multiple purposes. + CHARGE_LED_state = LED_STATE_OFF; +#endif } // If we want a LED to be dedicated to the simple hearbeat, we can use that instead of the charge LED #if defined(LED_HEARTBEAT) @@ -158,6 +164,7 @@ int32_t StatusLEDModule::runOnce() } } #endif +#ifdef LED_PAIRING if (!config.bluetooth.enabled || PAIRING_LED_starttime + 30 * 1000 < millis() || doing_fast_blink) { PAIRING_LED_state = LED_STATE_OFF; } else if (ble_state == unpaired) { @@ -172,6 +179,7 @@ int32_t StatusLEDModule::runOnce() } else { PAIRING_LED_state = LED_STATE_ON; } +#endif // Override if disabled in config if (config.device.led_heartbeat_disabled) { diff --git a/variants/esp32s3/ELECROW-ThinkNode-M5/variant.h b/variants/esp32s3/ELECROW-ThinkNode-M5/variant.h index 55bcd0dc7..2d0b413fe 100644 --- a/variants/esp32s3/ELECROW-ThinkNode-M5/variant.h +++ b/variants/esp32s3/ELECROW-ThinkNode-M5/variant.h @@ -18,7 +18,9 @@ #define BATTERY_PIN 8 #define ADC_CHANNEL ADC_CHANNEL_7 -#define ADC_MULTIPLIER 2.0 // 2.0 + 10% for correction of display undervoltage. +#define ADC_MULTIPLIER 2.0 + +#define OCV_ARRAY 4180, 4050, 3990, 3890, 3800, 3720, 3630, 3530, 3420, 3300, 3100 #define PIN_BUZZER 9 From 358e4e2fcd3fb800a9440ac50afab259b9cac01e Mon Sep 17 00:00:00 2001 From: HarukiToreda <116696711+HarukiToreda@users.noreply.github.com> Date: Wed, 17 Jun 2026 14:34:05 -0400 Subject: [PATCH 07/86] InkHUD: Wipe all messages option (#10721) * Wipe messages * trunk --- .../InkHUD/Applets/System/Menu/MenuAction.h | 1 + .../InkHUD/Applets/System/Menu/MenuApplet.cpp | 16 ++++++++++++++++ .../niche/InkHUD/Applets/System/Menu/MenuPage.h | 1 + src/graphics/niche/InkHUD/Persistence.cpp | 4 +++- 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/graphics/niche/InkHUD/Applets/System/Menu/MenuAction.h b/src/graphics/niche/InkHUD/Applets/System/Menu/MenuAction.h index 564afa558..4c70c434f 100644 --- a/src/graphics/niche/InkHUD/Applets/System/Menu/MenuAction.h +++ b/src/graphics/niche/InkHUD/Applets/System/Menu/MenuAction.h @@ -129,6 +129,7 @@ enum MenuAction { // Administration RESET_NODEDB_ALL, RESET_NODEDB_KEEP_FAVORITES, + WIPE_MESSAGES_ALL, }; } // namespace NicheGraphics::InkHUD diff --git a/src/graphics/niche/InkHUD/Applets/System/Menu/MenuApplet.cpp b/src/graphics/niche/InkHUD/Applets/System/Menu/MenuApplet.cpp index 12dde48e9..288dc0037 100644 --- a/src/graphics/niche/InkHUD/Applets/System/Menu/MenuApplet.cpp +++ b/src/graphics/niche/InkHUD/Applets/System/Menu/MenuApplet.cpp @@ -6,6 +6,7 @@ #include "GPS.h" #include "MeshRadio.h" #include "MeshService.h" +#include "MessageStore.h" #include "RTC.h" #include "Router.h" #include "airtime.h" @@ -1013,6 +1014,13 @@ void InkHUD::MenuApplet::execute(MenuItem item) rebootAtMsec = millis() + DEFAULT_REBOOT_SECONDS * 1000; break; + case WIPE_MESSAGES_ALL: + LOG_INFO("Wiping all messages from menu"); + messageStore.clearAllMessages(); + inkhud->persistence->loadLatestMessage(); + inkhud->forceUpdate(Drivers::EInk::UpdateTypes::FULL, true); + break; + default: LOG_WARN("Action not implemented"); } @@ -1130,6 +1138,7 @@ void InkHUD::MenuApplet::showPage(MenuPage page) // Administration Section items.push_back(MenuItem::Header("Administration")); items.push_back(MenuItem("Reset NodeDB", MenuPage::NODE_CONFIG_ADMIN_RESET)); + items.push_back(MenuItem("Wipe Messages", MenuPage::NODE_CONFIG_ADMIN_MESSAGES)); // Exit items.push_back(MenuItem("Exit", MenuPage::EXIT)); @@ -1534,6 +1543,13 @@ void InkHUD::MenuApplet::showPage(MenuPage page) items.push_back(MenuItem("Exit", MenuPage::EXIT)); break; + case NODE_CONFIG_ADMIN_MESSAGES: + previousPage = MenuPage::NODE_CONFIG; + items.push_back(MenuItem("Back", previousPage)); + items.push_back(MenuItem("Wipe All Messages", MenuAction::WIPE_MESSAGES_ALL, MenuPage::EXIT)); + items.push_back(MenuItem("Exit", MenuPage::EXIT)); + break; + // Exit case EXIT: sendToBackground(); // Menu applet dismissed, allow normal behavior to resume diff --git a/src/graphics/niche/InkHUD/Applets/System/Menu/MenuPage.h b/src/graphics/niche/InkHUD/Applets/System/Menu/MenuPage.h index 9bc6bb0cc..275bb97f2 100644 --- a/src/graphics/niche/InkHUD/Applets/System/Menu/MenuPage.h +++ b/src/graphics/niche/InkHUD/Applets/System/Menu/MenuPage.h @@ -36,6 +36,7 @@ enum MenuPage : uint8_t { NODE_CONFIG_BLUETOOTH, NODE_CONFIG_POSITION, NODE_CONFIG_ADMIN_RESET, + NODE_CONFIG_ADMIN_MESSAGES, TIMEZONE, APPLETS, AUTOSHOW, diff --git a/src/graphics/niche/InkHUD/Persistence.cpp b/src/graphics/niche/InkHUD/Persistence.cpp index 4197b81e0..8a8140bb3 100644 --- a/src/graphics/niche/InkHUD/Persistence.cpp +++ b/src/graphics/niche/InkHUD/Persistence.cpp @@ -22,6 +22,8 @@ void InkHUD::Persistence::loadSettings() // are immediately available to applets (DMApplet, AllMessageApplet, NotificationApplet). void InkHUD::Persistence::loadLatestMessage() { + latestMessage = LatestMessage(); + int lastBroadcastPos = -1, lastDMPos = -1, pos = 0; for (const StoredMessage &m : messageStore.getLiveMessages()) { if (m.type == MessageType::BROADCAST) { @@ -75,4 +77,4 @@ void InkHUD::Persistence::printSettings(Settings *settings) } */ -#endif \ No newline at end of file +#endif From 5ffd30c4a8ec05dc789172f1b6dc1bdf1a4c1168 Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Wed, 17 Jun 2026 13:34:29 -0500 Subject: [PATCH 08/86] Fix PKC on portduino sim by working around region blocks keygen and packet length (#10730) * Fix PKC on portduino sim by working around region blocks keygen and packet length * Reserve Compressed framing overhead in sim loopback capacity check The sim loopback re-encodes the Compressed wrapper back into decoded.payload.bytes (the same 233-byte field its data is copied from), so the carried bytes must leave room for protobuf framing. Checking against sizeof(c.data.bytes) (233) let near-max payloads overflow pb_encode_to_bytes(), which returns 0 and silently sends an empty loopback payload. Cap at sizeof(decoded.payload.bytes) minus the worst-case Compressed framing (meshtastic_Compressed_size - data size) for both the ciphertext and plaintext paths. Addresses Copilot review feedback on #10730. * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Run trunk fmt --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/mesh/NodeDB.cpp | 11 ++++- src/mesh/udp/UdpMulticastHandler.h | 3 ++ src/platform/portduino/SimRadio.cpp | 77 ++++++++++++++++++++++------- 3 files changed, 71 insertions(+), 20 deletions(-) diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index 01d554ab8..90eec34f8 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -3051,8 +3051,15 @@ bool NodeDB::checkLowEntropyPublicKey(const meshtastic_Config_SecurityConfig_pub bool NodeDB::generateCryptoKeyPair(const uint8_t *privateKey) { #if !(MESHTASTIC_EXCLUDE_PKI_KEYGEN || MESHTASTIC_EXCLUDE_PKI) - // Only generate keys for non-licensed users and if LoRa region is set - if (owner.is_licensed || config.lora.region == meshtastic_Config_LoRaConfig_RegionCode_UNSET) { + // Only generate keys for non-licensed users and if the LoRa region is set. The native simulator + // boots region-UNSET but still needs a keypair so PKI-encrypted DMs work between sim nodes, so + // allow keygen there regardless of region. + bool regionBlocksKeygen = config.lora.region == meshtastic_Config_LoRaConfig_RegionCode_UNSET; +#if ARCH_PORTDUINO + if (portduino_config.lora_module == use_simradio) + regionBlocksKeygen = false; +#endif + if (owner.is_licensed || regionBlocksKeygen) { return false; } diff --git a/src/mesh/udp/UdpMulticastHandler.h b/src/mesh/udp/UdpMulticastHandler.h index eb4d90d40..93306c41d 100644 --- a/src/mesh/udp/UdpMulticastHandler.h +++ b/src/mesh/udp/UdpMulticastHandler.h @@ -78,6 +78,9 @@ class UdpMulticastHandler final return; } mp.transport_mechanism = meshtastic_MeshPacket_TransportMechanism_TRANSPORT_MULTICAST_UDP; + // Preserve the whole MeshPacket as received: while payload_variant is encrypted, `channel` is a hash (and is 0 for + // PKI DMs), so it must be copied verbatim for the router to attempt PKI/channel decryption. Keep + // pki_encrypted/public_key too so downstream auth/metadata can reflect PKI usage correctly. UniquePacketPoolPacket p = packetPool.allocUniqueCopy(mp); // Unset received SNR/RSSI p->rx_snr = 0; diff --git a/src/platform/portduino/SimRadio.cpp b/src/platform/portduino/SimRadio.cpp index f9fabb617..c483eca02 100644 --- a/src/platform/portduino/SimRadio.cpp +++ b/src/platform/portduino/SimRadio.cpp @@ -209,16 +209,51 @@ void SimRadio::startSend(meshtastic_MeshPacket *txp) isReceiving = false; size_t numbytes = beginSending(txp); meshtastic_MeshPacket *p = packetPool.allocCopy(*txp); - perhapsDecode(p); - meshtastic_Compressed c = meshtastic_Compressed_init_default; - c.portnum = p->decoded.portnum; - // LOG_DEBUG("Send back to simulator with portNum %d", p->decoded.portnum); - if (p->decoded.payload.size <= sizeof(c.data.bytes)) { - memcpy(&c.data.bytes, p->decoded.payload.bytes, p->decoded.payload.size); - c.data.size = p->decoded.payload.size; - } else { - LOG_WARN("Payload size larger than compressed message allows! Send empty payload"); + + // A packet we originate that's encrypted for someone else (a PKI DM, channel == 0) can't be + // decrypted here. Attempting it only logs a spurious "no suitable channel" miss, and the + // ciphertext (up to MAX_LORA_PAYLOAD_LEN + MESHTASTIC_PKC_OVERHEAD) overflows the decoded + // loopback payload. Carry such packets as ciphertext instead so the receiving sim node can + // decrypt them as if they had arrived over the air (see unpackAndReceive()). + bool carryEncrypted = p->pki_encrypted; + if (!carryEncrypted) { + perhapsDecode(p); + // Channel packets we couldn't decrypt (e.g. relaying an unknown channel) are carried too. + carryEncrypted = (p->which_payload_variant == meshtastic_MeshPacket_encrypted_tag); } + + meshtastic_Compressed c = meshtastic_Compressed_init_default; + // The Compressed wrapper is re-encoded back into decoded.payload.bytes (the same 233-byte field + // its data is copied from), so the carried bytes must leave room for the protobuf framing or + // pb_encode_to_bytes() below overflows and silently drops the loopback payload. meshtastic_Compressed_size + // is the max encoded size for a full data field, so (meshtastic_Compressed_size - sizeof(c.data.bytes)) + // is the worst-case framing overhead to reserve. + constexpr size_t loopbackCapacity = sizeof(p->decoded.payload.bytes) - (meshtastic_Compressed_size - sizeof(c.data.bytes)); + if (carryEncrypted) { + // Sentinel portnum UNKNOWN_APP marks the payload as ciphertext for unpackAndReceive(). + c.portnum = meshtastic_PortNum_UNKNOWN_APP; + if (p->encrypted.size <= loopbackCapacity) { + memcpy(&c.data.bytes, p->encrypted.bytes, p->encrypted.size); + c.data.size = p->encrypted.size; + } else { + LOG_WARN("Encrypted payload (%u) exceeds sim loopback capacity (%u)! Send empty payload", (unsigned)p->encrypted.size, + (unsigned)loopbackCapacity); + } + } else { + c.portnum = p->decoded.portnum; + // LOG_DEBUG("Send back to simulator with portNum %d", p->decoded.portnum); + if (p->decoded.payload.size <= loopbackCapacity) { + memcpy(&c.data.bytes, p->decoded.payload.bytes, p->decoded.payload.size); + c.data.size = p->decoded.payload.size; + } else { + LOG_WARN("Payload size larger than compressed message allows! Send empty payload"); + } + } + + // pb_encode_to_bytes writes into decoded.payload, which aliases `encrypted` in the union, so all + // reads of p->encrypted above must be complete before this point. + p->decoded = meshtastic_Data_init_zero; + p->which_payload_variant = meshtastic_MeshPacket_decoded_tag; p->decoded.payload.size = pb_encode_to_bytes(p->decoded.payload.bytes, sizeof(p->decoded.payload.bytes), &meshtastic_Compressed_msg, &c); p->decoded.portnum = meshtastic_PortNum_SIMULATOR_APP; @@ -233,17 +268,23 @@ void SimRadio::unpackAndReceive(meshtastic_MeshPacket &p) { // Simulator packet (=Compressed packet) is encapsulated in a MeshPacket, so need to unwrap first meshtastic_Compressed scratch; - meshtastic_Compressed *decoded = NULL; if (p.which_payload_variant == meshtastic_MeshPacket_decoded_tag) { memset(&scratch, 0, sizeof(scratch)); - p.decoded.payload.size = - pb_decode_from_bytes(p.decoded.payload.bytes, p.decoded.payload.size, &meshtastic_Compressed_msg, &scratch); - if (p.decoded.payload.size) { - decoded = &scratch; - // Extract the original payload and replace - memcpy(&p.decoded.payload, &decoded->data, sizeof(decoded->data)); - // Switch the port from PortNum_SIMULATOR_APP back to the original PortNum - p.decoded.portnum = decoded->portnum; + if (pb_decode_from_bytes(p.decoded.payload.bytes, p.decoded.payload.size, &meshtastic_Compressed_msg, &scratch)) { + if (scratch.portnum == meshtastic_PortNum_UNKNOWN_APP) { + // The sender carried ciphertext verbatim (a packet it couldn't decrypt, e.g. a PKI DM, + // see startSend()). Restore it as an encrypted packet so the router decrypts it as if + // received over the air, instead of treating the ciphertext as a plaintext payload. The + // outer MeshPacket still carries from/to/id/channel/pki_encrypted, which decrypt needs. + p.which_payload_variant = meshtastic_MeshPacket_encrypted_tag; + memcpy(&p.encrypted.bytes, scratch.data.bytes, scratch.data.size); + p.encrypted.size = scratch.data.size; + } else { + // Extract the original payload and replace + memcpy(&p.decoded.payload, &scratch.data, sizeof(scratch.data)); + // Switch the port from PortNum_SIMULATOR_APP back to the original PortNum + p.decoded.portnum = scratch.portnum; + } } else LOG_ERROR("Error decoding proto for simulator message!"); } From 1cf6d7648f8ccb1354b0f6368fc8e0ae18e7b3b7 Mon Sep 17 00:00:00 2001 From: Jonathan Bennett Date: Wed, 17 Jun 2026 18:14:01 -0500 Subject: [PATCH 09/86] Leave src/platform out of the build filter by default (#10726) * Leave src/platform out of the build filter by default, and only add back what each build needs. * typo fix Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- platformio.ini | 2 +- variants/esp32/esp32-common.ini | 2 +- variants/esp32p4/esp32p4.ini | 2 +- variants/native/portduino.ini | 9 +++------ variants/nrf52840/nrf52.ini | 2 +- variants/nrf54l15/nrf54l15.ini | 4 ---- variants/rp2040/rp2040.ini | 2 +- variants/rp2350/rp2350.ini | 2 +- variants/stm32/stm32.ini | 2 +- 9 files changed, 10 insertions(+), 17 deletions(-) diff --git a/platformio.ini b/platformio.ini index bedd42a1e..99fc35d95 100644 --- a/platformio.ini +++ b/platformio.ini @@ -103,7 +103,7 @@ build_unflags = -std=gnu++11 build_flags = ${env.build_flags} -Os -std=gnu++17 -build_src_filter = ${env.build_src_filter} - - +build_src_filter = ${env.build_src_filter} - + - ; Common libs for communicating over TCP/IP networks such as MQTT [networking_base] diff --git a/variants/esp32/esp32-common.ini b/variants/esp32/esp32-common.ini index a94defff2..75aa8fc63 100644 --- a/variants/esp32/esp32-common.ini +++ b/variants/esp32/esp32-common.ini @@ -17,7 +17,7 @@ extra_scripts = extra_scripts/esp32_extra.py build_src_filter = - ${arduino_base.build_src_filter} - - - - - - + ${arduino_base.build_src_filter} + - - upload_speed = 921600 debug_init_break = tbreak setup diff --git a/variants/esp32p4/esp32p4.ini b/variants/esp32p4/esp32p4.ini index 7995a5205..64dbb2b58 100644 --- a/variants/esp32p4/esp32p4.ini +++ b/variants/esp32p4/esp32p4.ini @@ -13,7 +13,7 @@ build_flags = -DMESHTASTIC_EXCLUDE_PAXCOUNTER=1 build_src_filter = - ${esp32_common.build_src_filter} - - - - - - - + ${esp32_common.build_src_filter} - + - - extra_scripts = ${esp32_common.extra_scripts} diff --git a/variants/native/portduino.ini b/variants/native/portduino.ini index 9006f926a..699286e30 100644 --- a/variants/native/portduino.ini +++ b/variants/native/portduino.ini @@ -6,13 +6,10 @@ platform = framework = arduino build_src_filter = - ${env.build_src_filter} - - + ${env.build_src_filter} + - + + - - - - - - - - - - - + diff --git a/variants/nrf52840/nrf52.ini b/variants/nrf52840/nrf52.ini index f99c78d8a..f7f3d0885 100644 --- a/variants/nrf52840/nrf52.ini +++ b/variants/nrf52840/nrf52.ini @@ -43,7 +43,7 @@ build_unflags = -std=gnu++11 build_src_filter = - ${arduino_base.build_src_filter} - - - - - - - - - - - - + ${arduino_base.build_src_filter} + - - - - - - - - lib_deps= ${arduino_base.lib_deps} diff --git a/variants/nrf54l15/nrf54l15.ini b/variants/nrf54l15/nrf54l15.ini index d772de4d5..6b004a675 100644 --- a/variants/nrf54l15/nrf54l15.ini +++ b/variants/nrf54l15/nrf54l15.ini @@ -32,10 +32,6 @@ build_flags = build_src_filter = ${arduino_base.build_src_filter} - - - - - - - - - - - diff --git a/variants/rp2040/rp2040.ini b/variants/rp2040/rp2040.ini index 2b7002397..c560d21a6 100644 --- a/variants/rp2040/rp2040.ini +++ b/variants/rp2040/rp2040.ini @@ -20,7 +20,7 @@ build_flags = -D__FREERTOS=1 # -D _POSIX_THREADS build_src_filter = - ${arduino_base.build_src_filter} - - - - - - - - - - + ${arduino_base.build_src_filter} + - - - - - - lib_ignore = BluetoothOTA diff --git a/variants/rp2350/rp2350.ini b/variants/rp2350/rp2350.ini index 2931fd7b6..d2958d965 100644 --- a/variants/rp2350/rp2350.ini +++ b/variants/rp2350/rp2350.ini @@ -17,7 +17,7 @@ build_flags = -D__PLAT_RP2350__ -D__FREERTOS=1 build_src_filter = - ${arduino_base.build_src_filter} - - - - - - - - - - - - + ${arduino_base.build_src_filter} + - - - - - - - - lib_ignore = BluetoothOTA diff --git a/variants/stm32/stm32.ini b/variants/stm32/stm32.ini index 2116b046e..c5ef6c860 100644 --- a/variants/stm32/stm32.ini +++ b/variants/stm32/stm32.ini @@ -45,7 +45,7 @@ build_flags = -Wl,--wrap=_tzset_unlocked_r build_src_filter = - ${arduino_base.build_src_filter} - - - - - - - - - - - - - - - + ${arduino_base.build_src_filter} + - - - - - - - - - - board_upload.offset_address = 0x08000000 upload_protocol = stlink From ae3493a00c9f6403881cb505acacda5613b09b8f Mon Sep 17 00:00:00 2001 From: Jonathan Bennett Date: Wed, 17 Jun 2026 18:46:58 -0500 Subject: [PATCH 10/86] Set the time from GPS every 30 minutes (#10737) --- src/gps/GPS.cpp | 8 +++++--- src/gps/GPS.h | 1 + src/gps/RTC.cpp | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/gps/GPS.cpp b/src/gps/GPS.cpp index b2fbcac8d..5078a6a27 100644 --- a/src/gps/GPS.cpp +++ b/src/gps/GPS.cpp @@ -1160,7 +1160,10 @@ void GPS::setPowerState(GPSPowerState newState, uint32_t sleepTime) switch (newState) { case GPS_ACTIVE: case GPS_IDLE: - if (oldState == GPS_ACTIVE || oldState == GPS_IDLE) // If hardware already awake, no changes needed + if (oldState == GPS_ACTIVE) + break; + gotTime = false; + if (oldState == GPS_IDLE) // If hardware already awake, no changes needed break; if (oldState != GPS_ACTIVE && oldState != GPS_IDLE) // If hardware just waking now, clear buffer clearBuffer(); @@ -1484,8 +1487,7 @@ int32_t GPS::runOnce() // if gps_update_interval is <=10s, GPS never goes off, so we treat that differently uint32_t updateInterval = Default::getConfiguredOrDefaultMs(config.position.gps_update_interval); - // 1. Got a time for the first time - bool gotTime = (getRTCQuality() >= RTCQualityGPS); + // 1. Got a time for the first time this cycle if (!gotTime && lookForTime()) { // Note: we count on this && short-circuiting and not resetting the RTC time gotTime = true; } diff --git a/src/gps/GPS.h b/src/gps/GPS.h index 25de0379a..4028e351a 100644 --- a/src/gps/GPS.h +++ b/src/gps/GPS.h @@ -174,6 +174,7 @@ class GPS : private concurrency::OSThread uint32_t lastChecksumFailCount = 0; uint8_t currentStep = 0; int32_t currentDelay = 2000; + bool gotTime = false; #ifndef TINYGPS_OPTION_NO_CUSTOM_FIELDS // (20210908) TinyGps++ can only read the GPGSA "FIX TYPE" field diff --git a/src/gps/RTC.cpp b/src/gps/RTC.cpp index a8288a069..78439a50a 100644 --- a/src/gps/RTC.cpp +++ b/src/gps/RTC.cpp @@ -219,8 +219,8 @@ RTCSetResult perhapsSetRTC(RTCQuality q, const struct timeval *tv, bool forceUpd } else if (q == RTCQualityGPS) { shouldSet = true; LOG_DEBUG("Reapply GPS time: %ld secs", printableEpoch); - } else if (q == RTCQualityNTP && !Throttle::isWithinTimespanMs(lastSetMsec, (12 * 60 * 60 * 1000UL))) { - // Every 12 hrs we will slam in a new NTP or Phone GPS / NTP time, to correct for local RTC clock drift + } else if (q == RTCQualityNTP && !Throttle::isWithinTimespanMs(lastSetMsec, (30 * 60 * 1000UL))) { + // Every 30 minutes we will slam in a new NTP or Phone GPS / NTP time, to correct for local RTC clock drift shouldSet = true; LOG_DEBUG("Reapply external time to correct clock drift %ld secs", printableEpoch); } else { From b311e01ce09358eba5d683697b76a1bb1c4a4fdf Mon Sep 17 00:00:00 2001 From: Jonathan Bennett Date: Wed, 17 Jun 2026 22:16:09 -0500 Subject: [PATCH 11/86] Update OCV_ARRAY values in Thinknode m5 variant.h --- variants/esp32s3/ELECROW-ThinkNode-M5/variant.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/variants/esp32s3/ELECROW-ThinkNode-M5/variant.h b/variants/esp32s3/ELECROW-ThinkNode-M5/variant.h index 2d0b413fe..ee86c3df6 100644 --- a/variants/esp32s3/ELECROW-ThinkNode-M5/variant.h +++ b/variants/esp32s3/ELECROW-ThinkNode-M5/variant.h @@ -20,7 +20,7 @@ #define ADC_MULTIPLIER 2.0 -#define OCV_ARRAY 4180, 4050, 3990, 3890, 3800, 3720, 3630, 3530, 3420, 3300, 3100 +#define OCV_ARRAY 4100, 4050, 3990, 3890, 3800, 3720, 3630, 3530, 3420, 3300, 3100 #define PIN_BUZZER 9 From 79a7dcc46c63d507b551ddc2ac13a16067731b23 Mon Sep 17 00:00:00 2001 From: Tom <116762865+NomDeTom@users.noreply.github.com> Date: Thu, 18 Jun 2026 08:41:34 +0100 Subject: [PATCH 12/86] Pr1 nodedb warmstore (#10705) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * NodeDB: 3-tier node store with persistent warm tier (long-tail identity retention) Introduces a tiered NodeDB so the device retains identity (public key, last_heard) for far more nodes than fit in the full-record hot store, without growing heap or the persisted nodes.proto unboundedly. - Hot store: full NodeInfoLite, MAX_NUM_NODES (120 on nRF52). - Satellite maps: position/telemetry/environment/status capped at MAX_SATELLITE_NODES (40 freshest); eviction via enforceSatelliteCaps / evictSatelliteOverCap. - Warm tier (WarmNodeStore): 40 B {num,last_heard,public_key} records for evicted nodes so DMs to/from long-tail nodes keep encrypting/decrypting. Persisted to /prefs/warm.dat, or on nRF52840 a dedicated 12 KB raw-flash record-ring below LittleFS (3x4 KB pages; see linker scripts + the nrf52_warm_region.py post-link guard). NodeDB::getOrCreateMeshNode now demotes evicted nodes into the warm tier and re-admits them (restoring key/last_heard). Router PKI decrypt/encode resolve the peer key via NodeDB::copyPublicKey (hot store, then warm tier). NodeInfoLite gains snr_q4 (sint32, Q4-encoded dB); the float snr is zeroed on disk. NodeInfoLite grows 105 -> 112 B; backup 2432 -> 2468 B. Note: the snr_q4 .proto change still needs to land in the protobufs submodule (generated header is updated here; submodule pointer left at upstream). Co-Authored-By: Claude Opus 4.8 (1M context) * NodeDB: robust receive + retention for blocked (ignored) nodes Hardens how ignored/favourite nodes are received over admin and retained, closing paths where a block could be lost or accidentally cleared. - Blocking keeps the node's public key (admin set_ignored_node and addFromContact no longer zero it / drop the warm-tier key), so a blocked peer stays a verifiable identity. - set_ignored_node creates the node if absent, so a block by node ID sticks even for a node we've never heard from (e.g. pushed by a remote admin) with no NodeInfo or key. - Eviction protection (favourite/ignored/manually-verified) now also applies to the load-time hot-store migration and is never undone by cleanupMeshDB, which previously purged ignored nodes that lacked user info. - The hot-store migration leaves our own node (index 0) in place and prefers to demote non-protected nodes, like the runtime eviction scan. Caps the protected set (favourite + ignored + verified) at MAX_NUM_NODES-2 via NodeDB::setProtectedFlag(), so at least two evictable slots always remain and getOrCreateMeshNode can always make room — replacing the previous unconditional append that could run off the end of the node vector when every node was protected. A locally-set favourite/ignore that hits the cap reports back to the phone via a ClientNotification. Adds test_nodedb_blocked covering the migration, favourite/ignored eviction protection, ignored-survives-cleanup, and the protected-node cap. The maintenance methods stay private in production; the test reaches them through a PIO_UNIT_TESTING-guarded friend shim. Co-Authored-By: Claude Opus 4.8 (1M context) # Conflicts: # src/mesh/NodeDB.h * fix copilot comments * once again * WarmNodeStore: fix cppcheck warnings (uninitvar, constVariablePointer) Zero-initialise `stranded[]` and `seqs[]/order[]` VLAs so cppcheck can verify there are no unguarded reads of uninitialised memory (the guards exist but are not visible to static analysis). Mark two local pointers `const` where the pointed-to entry is never mutated after assignment. Co-Authored-By: Claude Sonnet 4.6 * self-care added to assist 2.7 and 2.8 nodedb migration * Tidy warm-store/self-care: comments, guards, log + flash cleanup Style/cleanup pass over the branch (no behavior change except the noted preprocessor simplifications, which are semantically identical): - Comments: move function descriptions to the headers, cap in-function comments at ~3-4 lines, drop leading-number step markers, label stacked #endif blocks, de-decorate banner comments. - dumpToLog: fully gate decl + definition + AdminModule call site behind MESHTASTIC_NODEDB_MIGRATION_VERBOSE so it compiles out when disabled (~1.2 KB when off). - mesh-pb-constants: drop the dead nRF52832 WARM_NODE_COUNT branch and trim the macro docs. - WarmNodeStore: simplify the redundant `ARCH_NRF52 && NRF52840_XXAA` guards to `NRF52840_XXAA`, add a kNoPage sentinel for the ring page state. - Shorten the always-on LOG_WARN strings (~120 B flash). Co-Authored-By: Claude Opus 4.8 (1M context) * more tidying up, aligning with docs and undoing other-arch regressions * Update protobufs (#19) Co-authored-by: NomDeTom <116762865+NomDeTom@users.noreply.github.com> * made the migration pathway cleareer * address copilot review * fixed a copilot review on a downstream PR. * Address Copilot review comments for PR #10705 (warmstore/nodedb) - WarmNodeStore.h: default MIGRATION_VERBOSE to 0 (suppress info-level chatter on production builds; opt in with =1) - WarmNodeStore.cpp load(): move memset to top of function so all failure paths (header-read fail, invalid header) leave entries clear - WarmNodeStore.cpp save(): replace manual spiLock lock/unlock around mkdir with LockGuard covering the full SafeFile sequence, matching the lock discipline in load() - Router.cpp: memcpy(&p->public_key.bytes, ...) -> memcpy(p->public_key.bytes, ...) — pass decayed uint8_t* rather than pointer-to-array - AdminModule.cpp: check setProtectedFlag return for PKC auto-favorite; log cap-refusal warning instead of unconditional "auto-favoriting" - nrf52_warm_region.py: error message references both v6.ld and v7.ld Co-Authored-By: Claude Sonnet 4.6 * NodeDB: formatting cleanup (blank lines after preprocessor blocks) Co-Authored-By: Claude Sonnet 4.6 * Lukewarm store --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Ben Meadors --- .github/copilot-instructions.md | 19 +- extra_scripts/nrf52_warm_region.py | 70 ++++ src/graphics/draw/MenuHandler.cpp | 17 +- src/mesh/NodeDB.cpp | 463 +++++++++++++++++++-- src/mesh/NodeDB.h | 86 +++- src/mesh/NodeDBLegacyMigration.cpp | 3 + src/mesh/Router.cpp | 30 +- src/mesh/WarmNodeStore.cpp | 548 +++++++++++++++++++++++++ src/mesh/WarmNodeStore.h | 130 ++++++ src/mesh/mesh-pb-constants.h | 66 ++- src/modules/AdminModule.cpp | 41 +- src/platform/nrf52/nrf52840_s140_v6.ld | 46 +++ src/platform/nrf52/nrf52840_s140_v7.ld | 7 +- test/test_nodedb_blocked/test_main.cpp | 193 +++++++++ test/test_warm_store/test_main.cpp | 211 ++++++++++ variants/nrf52840/nrf52.ini | 1 + variants/nrf52840/nrf52840.ini | 4 + 17 files changed, 1848 insertions(+), 87 deletions(-) create mode 100644 extra_scripts/nrf52_warm_region.py create mode 100644 src/mesh/WarmNodeStore.cpp create mode 100644 src/mesh/WarmNodeStore.h create mode 100644 src/platform/nrf52/nrf52840_s140_v6.ld create mode 100644 test/test_nodedb_blocked/test_main.cpp create mode 100644 test/test_warm_store/test_main.cpp diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 0e2f179ca..404eb021c 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -191,7 +191,24 @@ Writers go through `setNodeStatus`, `updatePosition`, `updateTelemetry` (which d ### Eviction -Every code path that drops a node from the header table must also evict the satellites. The single chokepoint is `eraseNodeSatellites(NodeNum)`; it's already called from `getOrCreateMeshNode`'s oldest-boring eviction, `removeNodeByNum`, both branches of `resetNodes`, `cleanupMeshDB`, `addFromContact`'s ignored-branch, and `AdminModule`'s `set_ignored_node`. Add new eviction sites here, not by calling `.erase()` directly. +Every code path that drops a node from the header table must also evict the satellites. The single chokepoint is `eraseNodeSatellites(NodeNum)`; it's already called from `getOrCreateMeshNode`'s oldest-boring eviction, `demoteOldestHotNodesToWarm` (the over-cap warm-tier migration), `removeNodeByNum`, both branches of `resetNodes`, `cleanupMeshDB`, `addFromContact`'s ignored-branch, and `AdminModule`'s `set_ignored_node`. Add new eviction sites here, not by calling `.erase()` directly. (Note: `enforceSatelliteCaps`/`evictSatelliteOverCap` call `.erase()` directly on purpose — that's a satellite-only cap trim where the node _stays_ in the header, a different operation from this chokepoint.) + +### Warm tier (long-tail identity) + +On every arch except STM32WL and bare nRF52832 (`WARM_NODE_COUNT > 0`), a node evicted from the header table is not forgotten outright: `WarmNodeStore` (`src/mesh/WarmNodeStore.{h,cpp}`) keeps a 40 B `{num, last_heard, public_key}` record per evicted node — primarily so PKI DMs to/from a long-tail node keep decrypting without re-running a NodeInfo exchange (the rest of `NodeInfoLite` rebuilds from traffic in seconds). + +- **Write:** `getOrCreateMeshNode`'s eviction and `demoteOldestHotNodesToWarm` (the over-cap boot migration) call `warmStore.absorb(num, last_heard, key)` _before_ the node leaves the header. +- **Read-back:** `getOrCreateMeshNode` calls `warmStore.take()` to rehydrate `last_heard` + key when a warm node is re-admitted; `copyPublicKey()` falls back to the warm tier so the PKI send path finds keys for evicted peers. +- **Persistence:** nRF52840 uses a 12 KB raw-flash record-ring at `0xEA000` (below LittleFS; append + replay + compact-on-rotate, link-guarded by `nrf52840_s140_v7.ld` and `extra_scripts/nrf52_warm_region.py`). Everywhere else: a `/prefs/warm.dat` snapshot flushed by `saveIfDirty()` on the node-DB save cadence. +- **Tunables** (`mesh-pb-constants.h`): `WARM_NODE_COUNT` (per-arch; `0` disables the tier) and `MAX_NUM_NODES` (hot cap — 120 on nRF52840/generic ESP32 to fit the 28 KB LittleFS; ESP32-S3 keeps its flash-scaled 100/200/250, portduino 250). Verbose migration/self-care tracing routes through `LOG_MIGRATION`, gated by `MESHTASTIC_NODEDB_MIGRATION_VERBOSE`. + +### Satellite caps + +Only the freshest `MAX_SATELLITE_NODES` nodes keep satellite payloads; the rest of the header table carries just the `NodeInfoLite`. The cap is **per-platform**: 40 on RAM-constrained parts (nRF52840, generic ESP32) since the four maps live in internal SRAM (not PSRAM, ~408 B/node across the four), and 250 on flash-rich hosts (ESP32-S3, portduino) so every hot node can carry rich data as before the cap existed. `enforceSatelliteCaps()` trims each map to the cap on load (returns whether it trimmed); `evictSatelliteOverCap()` trims before each insert. Eviction is by the owning node's hot `last_heard` (stalest first, demoted/absent nodes rank as `last_heard==0`); self is never trimmed. + +### On-boot self-care + +`NodeDB::nodeDBSelfCare()` runs once identity is established (the constructor after key (re)gen, and `reloadFromDisk()` — _not_ inside `loadFromDisk`, where `getNodeNum()` is still 0). It confirms self is present (warns if a non-empty DB is missing us — a foreign/over-cap file), pins self to index 0, demotes/trims only **non-self** overflow into the warm tier, then rewrites `nodes.proto` **once** and only if it healed something — and never while encrypted storage is locked (it would persist placeholder defaults). `loadFromDisk` deliberately leaves the loaded store untrimmed for this pass. ### Sync flow: thin NodeInfo + post-COMPLETE_ID replay (no opt-in) diff --git a/extra_scripts/nrf52_warm_region.py b/extra_scripts/nrf52_warm_region.py new file mode 100644 index 000000000..e1a942bef --- /dev/null +++ b/extra_scripts/nrf52_warm_region.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# trunk-ignore-all(ruff/F821) +# trunk-ignore-all(flake8/F821): For SConstruct imports +# +# Post-link guard for the warm-node-store raw-flash region on nRF52840. +# +# The 3 app-region pages below LittleFS (0xEA000-0xED000, reclaimed by whole-image +# LTO) are reserved for the WarmNodeStore record-ring (see WarmNodeStore.h). Our +# linker scripts (nrf52840_s140_v6.ld and nrf52840_s140_v7.ld) cap the image at +# 0xEA000, but boards on the framework-default script (FLASH ending at 0xED000) could +# silently place code in those pages — the first warm-store save would then brick the +# device. This turns that into a build failure. +# +# Image flash end = __etext + sizeof(.data) (loaded at LMA __etext); symbols from +# the framework's nrf52_common.ld. +import os + +Import("env") + +WARM_REGION_BASE = 0xEA000 # keep in sync with WARM_FLASH_REGION_BASE in WarmNodeStore.h (3 x 4 KB record-ring) + +_tc = env.PioPlatform().get_package_dir("toolchain-gccarmnoneeabi") or "" +_NM = os.path.join(_tc, "bin", "arm-none-eabi-nm") +if not os.path.isfile(_NM): + _NM = "arm-none-eabi-nm" # fall back to PATH + + +def _assert_warm_region_clear(source, target, env): + import subprocess + import sys + + try: + elf = env.subst("$BUILD_DIR/${PROGNAME}.elf") + out = subprocess.check_output([_NM, elf], universal_newlines=True) + except Exception as exc: # tooling hiccup: warn loudly, don't wedge the build + print("nrf52_warm_region: WARNING - guard skipped (nm failed: %s)" % exc) + return + + syms = {} + for line in out.split("\n"): + f = line.split() + if len(f) >= 3 and f[-1] in ("__etext", "__data_start__", "__data_end__"): + syms[f[-1]] = int(f[0], 16) + if len(syms) != 3: + print("nrf52_warm_region: WARNING - guard skipped (linker symbols not found)") + return + + flash_end = syms["__etext"] + (syms["__data_end__"] - syms["__data_start__"]) + if flash_end > WARM_REGION_BASE: + sys.stderr.write( + "\n*** nrf52 warm-region guard: image ends at 0x%X, past the reserved " + "warm-store region at 0x%X ***\n" + "The 12 KB region at 0xEA000 holds the WarmNodeStore record-ring; a warm-store\n" + "save would overwrite this firmware's tail. Shrink the image, or shrink/move\n" + "the region (WARM_FLASH_REGION_BASE in src/mesh/WarmNodeStore.h, the FLASH\n" + "LENGTH in src/platform/nrf52/nrf52840_s140_v6.ld and _v7.ld, and this guard).\n\n" + % (flash_end, WARM_REGION_BASE) + ) + from SCons.Script import Exit + + Exit(1) + print( + "nrf52_warm_region: guard OK -- image ends at 0x%X, %d KB clear of the warm region" + % (flash_end, (WARM_REGION_BASE - flash_end) // 1024) + ) + + +# Attach to the phony "buildprog" alias (not the .elf node) so the guard runs +# on incremental relinks too -- same reasoning as nrf52_lto.py's guard. +env.AddPostAction("buildprog", _assert_warm_region_clear) diff --git a/src/graphics/draw/MenuHandler.cpp b/src/graphics/draw/MenuHandler.cpp index 365739112..0f4d8d713 100644 --- a/src/graphics/draw/MenuHandler.cpp +++ b/src/graphics/draw/MenuHandler.cpp @@ -1572,6 +1572,7 @@ void menuHandler::manageNodeMenu() nodeDB->set_favorite(false, menuHandler::pickedNodeNum); } else { LOG_INFO("Adding node %08X to favorites", menuHandler::pickedNodeNum); + // set_favorite() already logs PROTECTED_CAP_WARN_FMT on a cap refusal; don't double-log here. nodeDB->set_favorite(true, menuHandler::pickedNodeNum); } screen->setFrames(graphics::Screen::FOCUS_PRESERVE); @@ -1615,15 +1616,23 @@ void menuHandler::manageNodeMenu() return; } + bool changed = false; if (nodeInfoLiteIsIgnored(n)) { nodeInfoLiteSetBit(n, NODEINFO_BITFIELD_IS_IGNORED_MASK, false); LOG_INFO("Unignoring node %08X", menuHandler::pickedNodeNum); - } else { - nodeInfoLiteSetBit(n, NODEINFO_BITFIELD_IS_IGNORED_MASK, true); + changed = true; + } else if (nodeDB->setProtectedFlag(n, NODEINFO_BITFIELD_IS_IGNORED_MASK, true)) { LOG_INFO("Ignoring node %08X", menuHandler::pickedNodeNum); + changed = true; + } else { + LOG_WARN(NodeDB::PROTECTED_CAP_WARN_FMT, "ignore", menuHandler::pickedNodeNum, MAX_NUM_NODES - 2); + } + // Only persist/notify when the ignore bit actually moved; a cap + // refusal changed nothing and shouldn't trigger a prefs save. + if (changed) { + nodeDB->notifyObservers(true); + nodeDB->saveToDisk(); } - nodeDB->notifyObservers(true); - nodeDB->saveToDisk(); screen->setFrames(graphics::Screen::FOCUS_PRESERVE); return; } diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index 90eec34f8..38025ad10 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -197,6 +197,8 @@ bool meshtastic_NodeDatabase_callback(pb_istream_t *istream, pb_ostream_t *ostre if (ostream) { const auto *vec = static_cast *>(iter->pData); for (auto item : *vec) { + item.snr_q4 = (int32_t)(item.snr * 4.0f); + item.snr = 0.0f; if (!pb_encode_tag_for_field(ostream, iter)) return false; if (!pb_encode_submessage(ostream, meshtastic_NodeInfoLite_fields, &item)) @@ -206,8 +208,12 @@ bool meshtastic_NodeDatabase_callback(pb_istream_t *istream, pb_ostream_t *ostre if (istream && istream->bytes_left) { meshtastic_NodeInfoLite node = meshtastic_NodeInfoLite_init_zero; auto *vec = static_cast *>(iter->pData); - if (pb_decode(istream, meshtastic_NodeInfoLite_fields, &node)) + if (pb_decode(istream, meshtastic_NodeInfoLite_fields, &node)) { + if (node.snr_q4) + node.snr = node.snr_q4 / 4.0f; + node.snr_q4 = 0; vec->push_back(node); + } } return true; } @@ -482,9 +488,17 @@ NodeDB::NodeDB() myNodeInfo.my_node_num = crc32Buffer(config.security.public_key.bytes, config.security.public_key.size); } #endif - // Include our owner in the node db under our nodenum - meshtastic_NodeInfoLite *info = getOrCreateMeshNode(getNodeNum()); - TypeConversions::CopyUserToNodeInfoLite(info, owner); + // Identity is now established, so run the self-care pass on the store + // loadFromDisk() deliberately left untrimmed: confirm self, trim/demote only + // non-self overflow, pin self to index 0, rewrite once if healed. + nodeDBSelfCare(); + + // If we migrated from legacy during loadFromDisk(), persist the migrated DB + // only after identity and self-care are established. + if (migrationSavePending) { + saveNodeDatabaseToDisk(); + migrationSavePending = false; + } // If node database has not been saved for the first time, save it now #ifdef FSCom @@ -689,6 +703,39 @@ template std::vector snapshotSatelliteNodeNums(const Map } return result; } + +// Drop the stalest entry of `map` (staleness proxied via the owner's +// last_heard; 0 = owner evicted, i.e. an orphan — first out). Never evicts our +// own node's entry. Caller holds satelliteMutex. Returns false if nothing +// could be evicted. +template bool evictStalestSatellite(NodeDB &db, Map &map) +{ + auto victim = map.end(); + uint32_t victimTs = UINT32_MAX; + for (auto it = map.begin(); it != map.end(); ++it) { + if (it->first == db.getNodeNum()) + continue; + uint32_t ts = db.hotNodeLastHeard(it->first); + if (ts < victimTs) { + victimTs = ts; + victim = it; + } + } + if (victim == map.end()) + return false; + map.erase(victim); + return true; +} + +// Keep `map` within MAX_SATELLITE_NODES ahead of inserting `incoming` (the +// tier-1/tier-2 split: only the freshest MAX_SATELLITE_NODES nodes carry +// satellite payloads). Caller holds satelliteMutex. +template void evictSatelliteOverCap(NodeDB &db, Map &map, NodeNum incoming) +{ + if (map.size() < MAX_SATELLITE_NODES || map.count(incoming)) + return; + evictStalestSatellite(db, map); +} } // namespace void NodeDB::resetRadioConfig(bool is_fresh_install) @@ -721,6 +768,7 @@ bool NodeDB::factoryReset(bool eraseBleBonds) LOG_ERROR("Could not remove rangetest.csv file"); } #endif + spiLock->unlock(); // rmDir above nuked the .dat file, but TransmitHistory's in-memory @@ -731,6 +779,14 @@ bool NodeDB::factoryReset(bool eraseBleBonds) #if HAS_SCREEN messageStore.clearAllMessages(); #endif + +#if WARM_NODE_COUNT > 0 + // On nRF52840 the warm tier lives in raw flash outside /prefs, so rmDir + // didn't touch it; clear it and persist the empty store. + warmStore.clear(); + warmStore.saveIfDirty(); +#endif + // second, install default state (this will deal with the duplicate mac address issue) installDefaultNodeDatabase(); installDefaultDeviceState(); @@ -745,6 +801,7 @@ bool NodeDB::factoryReset(bool eraseBleBonds) // This will erase what's in NVS including ssl keys, persistent variables and ble pairing nvs_flash_erase(); #endif + #ifdef ARCH_NRF52 LOG_INFO("Clear bluetooth bonds!"); bond_print_list(BLE_GAP_ROLE_PERIPH); @@ -767,12 +824,15 @@ void NodeDB::installDefaultNodeDatabase() #if !MESHTASTIC_EXCLUDE_POSITIONDB nodePositions.clear(); #endif + #if !MESHTASTIC_EXCLUDE_TELEMETRYDB nodeTelemetry.clear(); #endif + #if !MESHTASTIC_EXCLUDE_ENVIRONMENTDB nodeEnvironment.clear(); #endif + #if !MESHTASTIC_EXCLUDE_STATUSDB nodeStatus.clear(); #endif @@ -835,34 +895,42 @@ void NodeDB::installDefaultConfig(bool preserveKey = false) #else config.lora.region = meshtastic_Config_LoRaConfig_RegionCode_UNSET; #endif + #ifdef USERPREFS_LORACONFIG_MODEM_PRESET config.lora.modem_preset = USERPREFS_LORACONFIG_MODEM_PRESET; #else config.lora.modem_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_FAST; #endif + #ifdef USERPREFS_LORACONFIG_USE_PRESET config.lora.use_preset = USERPREFS_LORACONFIG_USE_PRESET; #else config.lora.use_preset = true; #endif + #ifdef USERPREFS_LORACONFIG_BANDWIDTH config.lora.bandwidth = USERPREFS_LORACONFIG_BANDWIDTH; #endif + #ifdef USERPREFS_LORACONFIG_SPREAD_FACTOR config.lora.spread_factor = USERPREFS_LORACONFIG_SPREAD_FACTOR; #endif + #ifdef USERPREFS_LORACONFIG_CODING_RATE config.lora.coding_rate = USERPREFS_LORACONFIG_CODING_RATE; #endif + #ifdef USERPREFS_LORACONFIG_OVERRIDE_FREQUENCY config.lora.override_frequency = USERPREFS_LORACONFIG_OVERRIDE_FREQUENCY; #endif + config.lora.hop_limit = HOP_RELIABLE; #ifdef USERPREFS_CONFIG_LORA_IGNORE_MQTT config.lora.ignore_mqtt = USERPREFS_CONFIG_LORA_IGNORE_MQTT; #else config.lora.ignore_mqtt = false; #endif + // Initialize admin_key_count to zero byte numAdminKeys = 0; @@ -892,6 +960,7 @@ void NodeDB::installDefaultConfig(bool preserveKey = false) numAdminKeys++; } #endif + config.security.admin_key_count = numAdminKeys; if (shouldPreserveKey) { @@ -906,6 +975,7 @@ void NodeDB::installDefaultConfig(bool preserveKey = false) #ifdef PIN_GPS_EN config.position.gps_en_gpio = PIN_GPS_EN; #endif + #if defined(USERPREFS_CONFIG_GPS_MODE) config.position.gps_mode = USERPREFS_CONFIG_GPS_MODE; #elif !HAS_GPS || GPS_DEFAULT_NOT_PRESENT @@ -918,11 +988,13 @@ void NodeDB::installDefaultConfig(bool preserveKey = false) #else config.position.gps_mode = meshtastic_Config_PositionConfig_GpsMode_ENABLED; #endif + #ifdef USERPREFS_CONFIG_SMART_POSITION_ENABLED config.position.position_broadcast_smart_enabled = USERPREFS_CONFIG_SMART_POSITION_ENABLED; #else config.position.position_broadcast_smart_enabled = true; #endif + config.position.broadcast_smart_minimum_distance = 100; config.position.broadcast_smart_minimum_interval_secs = default_broadcast_smart_minimum_interval_secs; if (config.device.role != meshtastic_Config_DeviceConfig_Role_ROUTER && @@ -942,6 +1014,7 @@ void NodeDB::installDefaultConfig(bool preserveKey = false) // default to bluetooth capability of platform as default config.bluetooth.enabled = true; #endif + config.bluetooth.fixed_pin = defaultBLEPin; #if defined(ST7735_CS) || defined(USE_EINK) || defined(ILI9341_DRIVER) || defined(ILI9342_DRIVER) || defined(ST7789_CS) || \ @@ -953,7 +1026,7 @@ void NodeDB::installDefaultConfig(bool preserveKey = false) if (st7789_id == 0xFFFFFF) { hasScreen = false; } -#endif +#endif // HELTEC_MESH_NODE_T114 #elif ARCH_PORTDUINO bool hasScreen = false; if (portduino_config.displayPanel) @@ -973,6 +1046,7 @@ void NodeDB::installDefaultConfig(bool preserveKey = false) config.bluetooth.mode = hasScreen ? meshtastic_Config_BluetoothConfig_PairingMode_RANDOM_PIN : meshtastic_Config_BluetoothConfig_PairingMode_FIXED_PIN; #endif + // for backward compat, default position flags are ALT+MSL config.position.position_flags = (meshtastic_Config_PositionConfig_PositionFlags_ALTITUDE | meshtastic_Config_PositionConfig_PositionFlags_ALTITUDE_MSL | @@ -985,8 +1059,8 @@ void NodeDB::installDefaultConfig(bool preserveKey = false) config.network.enabled_protocols = USERPREFS_NETWORK_ENABLED_PROTOCOLS; #else config.network.enabled_protocols = 0; -#endif -#endif +#endif // Network enabled protocols +#endif // UDP Multicast #ifdef USERPREFS_NETWORK_WIFI_ENABLED config.network.wifi_enabled = USERPREFS_NETWORK_WIFI_ENABLED; @@ -1013,16 +1087,20 @@ void NodeDB::installDefaultConfig(bool preserveKey = false) #ifdef DISPLAY_FLIP_SCREEN config.display.flip_screen = true; #endif + #ifdef RAK4630 config.display.wake_on_tap_or_motion = true; #endif + #if defined(T_WATCH_S3) || defined(SENSECAP_INDICATOR) config.display.screen_on_secs = 30; config.display.wake_on_tap_or_motion = true; #endif + #ifdef COMPASS_ORIENTATION config.display.compass_orientation = COMPASS_ORIENTATION; #endif + #if defined(ARCH_ESP32) && !MESHTASTIC_EXCLUDE_WIFI if (MeshtasticOTA::isUpdated()) { MeshtasticOTA::recoverConfig(&config.network); @@ -1046,6 +1124,7 @@ void NodeDB::initConfigIntervals() #else config.position.gps_update_interval = default_gps_update_interval; #endif + #ifdef USERPREFS_CONFIG_POSITION_BROADCAST_INTERVAL config.position.position_broadcast_secs = USERPREFS_CONFIG_POSITION_BROADCAST_INTERVAL; #else @@ -1082,22 +1161,26 @@ void NodeDB::installDefaultModuleConfig() defined(NEOPIXEL_STATUS_NOTIFICATION_PIN) moduleConfig.external_notification.enabled = true; #endif + #if defined(PIN_BUZZER) moduleConfig.external_notification.output_buzzer = PIN_BUZZER; moduleConfig.external_notification.use_pwm = true; moduleConfig.external_notification.alert_message_buzzer = true; #endif + #if defined(PIN_VIBRATION) moduleConfig.external_notification.output_vibra = PIN_VIBRATION; moduleConfig.external_notification.alert_message_vibra = true; moduleConfig.external_notification.output_ms = 500; #endif + #if defined(LED_NOTIFICATION) moduleConfig.external_notification.output = LED_NOTIFICATION; moduleConfig.external_notification.active = LED_STATE_ON; moduleConfig.external_notification.alert_message = true; moduleConfig.external_notification.output_ms = 1000; #endif + #if defined(PIN_VIBRATION) moduleConfig.external_notification.nag_timeout = 2; #elif defined(PIN_BUZZER) || defined(LED_NOTIFICATION) || defined(NEOPIXEL_STATUS_NOTIFICATION_PIN) @@ -1114,14 +1197,16 @@ void NodeDB::installDefaultModuleConfig() moduleConfig.external_notification.nag_timeout = 0; #else moduleConfig.external_notification.nag_timeout = default_ringtone_nag_secs; -#endif -#endif +#endif // HAS_TFT +#endif // HAS_I2S + #ifdef NANO_G2_ULTRA moduleConfig.external_notification.enabled = true; moduleConfig.external_notification.alert_message = true; moduleConfig.external_notification.output_ms = 100; moduleConfig.external_notification.active = true; -#endif +#endif // NANO_G2_ULTRA + #ifdef T_LORA_PAGER moduleConfig.canned_message.updown1_enabled = true; moduleConfig.canned_message.inputbroker_pin_a = ROTARY_A; @@ -1131,35 +1216,43 @@ void NodeDB::installDefaultModuleConfig() moduleConfig.canned_message.inputbroker_event_ccw = meshtastic_ModuleConfig_CannedMessageConfig_InputEventChar(29); moduleConfig.canned_message.inputbroker_event_press = meshtastic_ModuleConfig_CannedMessageConfig_InputEventChar_SELECT; #endif + moduleConfig.has_canned_message = true; + #if USERPREFS_MQTT_ENABLED && !MESHTASTIC_EXCLUDE_MQTT moduleConfig.mqtt.enabled = true; #endif + #ifdef USERPREFS_MQTT_ADDRESS strncpy(moduleConfig.mqtt.address, USERPREFS_MQTT_ADDRESS, sizeof(moduleConfig.mqtt.address)); #else strncpy(moduleConfig.mqtt.address, default_mqtt_address, sizeof(moduleConfig.mqtt.address)); #endif + #ifdef USERPREFS_MQTT_USERNAME strncpy(moduleConfig.mqtt.username, USERPREFS_MQTT_USERNAME, sizeof(moduleConfig.mqtt.username)); #else strncpy(moduleConfig.mqtt.username, default_mqtt_username, sizeof(moduleConfig.mqtt.username)); #endif + #ifdef USERPREFS_MQTT_PASSWORD strncpy(moduleConfig.mqtt.password, USERPREFS_MQTT_PASSWORD, sizeof(moduleConfig.mqtt.password)); #else strncpy(moduleConfig.mqtt.password, default_mqtt_password, sizeof(moduleConfig.mqtt.password)); #endif + #ifdef USERPREFS_MQTT_ROOT_TOPIC strncpy(moduleConfig.mqtt.root, USERPREFS_MQTT_ROOT_TOPIC, sizeof(moduleConfig.mqtt.root)); #else strncpy(moduleConfig.mqtt.root, default_mqtt_root, sizeof(moduleConfig.mqtt.root)); #endif + #ifdef USERPREFS_MQTT_ENCRYPTION_ENABLED moduleConfig.mqtt.encryption_enabled = USERPREFS_MQTT_ENCRYPTION_ENABLED; #else moduleConfig.mqtt.encryption_enabled = default_mqtt_encryption_enabled; #endif + #ifdef USERPREFS_MQTT_TLS_ENABLED moduleConfig.mqtt.tls_enabled = USERPREFS_MQTT_TLS_ENABLED; #else @@ -1253,11 +1346,13 @@ void NodeDB::initModuleConfigIntervals() #else moduleConfig.telemetry.device_update_interval = MAX_INTERVAL; #endif + #ifdef USERPREFS_CONFIG_ENV_TELEM_UPDATE_INTERVAL moduleConfig.telemetry.environment_update_interval = USERPREFS_CONFIG_ENV_TELEM_UPDATE_INTERVAL; #else moduleConfig.telemetry.environment_update_interval = 0; #endif + #ifdef USERPREFS_CONFIG_ENVIRONMENT_MEASUREMENT_ENABLED moduleConfig.telemetry.environment_measurement_enabled = USERPREFS_CONFIG_ENVIRONMENT_MEASUREMENT_ENABLED; #endif @@ -1302,6 +1397,10 @@ void NodeDB::resetNodes(bool keepFavorites) std::fill(nodeDatabase.nodes.begin() + 1, nodeDatabase.nodes.end(), meshtastic_NodeInfoLite()); } (void)ourNum; +#if WARM_NODE_COUNT > 0 + warmStore.clear(); // warm entries are never favorites; a DB reset clears them too +#endif + devicestate.has_rx_waypoint = false; saveNodeDatabaseToDisk(); saveDeviceStateToDisk(); @@ -1323,6 +1422,11 @@ void NodeDB::removeNodeByNum(NodeNum nodeNum) meshtastic_NodeInfoLite()); if (removed) eraseNodeSatellites(nodeNum); +#if WARM_NODE_COUNT > 0 + // Explicit user removal: don't let the warm tier resurrect the node + warmStore.remove(nodeNum); +#endif + LOG_DEBUG("NodeDB::removeNodeByNum purged %d entries. Save changes", removed); saveNodeDatabaseToDisk(); } @@ -1436,6 +1540,7 @@ void NodeDB::setNodeStatus(NodeNum n, const meshtastic_StatusMessage &status) (void)status; #else concurrency::LockGuard guard(&satelliteMutex); + evictSatelliteOverCap(*this, nodeStatus, n); nodeStatus[n] = status; #endif } @@ -1447,6 +1552,7 @@ void NodeDB::touchNodePositionTime(NodeNum n, uint32_t time) (void)time; #else concurrency::LockGuard guard(&satelliteMutex); + evictSatelliteOverCap(*this, nodePositions, n); nodePositions[n].time = time; #endif } @@ -1457,23 +1563,65 @@ void NodeDB::eraseNodeSatellites(NodeNum n) #if !MESHTASTIC_EXCLUDE_POSITIONDB nodePositions.erase(n); #endif + #if !MESHTASTIC_EXCLUDE_TELEMETRYDB nodeTelemetry.erase(n); #endif + #if !MESHTASTIC_EXCLUDE_ENVIRONMENTDB nodeEnvironment.erase(n); #endif + #if !MESHTASTIC_EXCLUDE_STATUSDB nodeStatus.erase(n); #endif } +bool NodeDB::enforceSatelliteCaps() +{ + concurrency::LockGuard guard(&satelliteMutex); + bool trimmedAny = false; + auto trim = [this, &trimmedAny](auto &map, const char *name) { + const size_t before = map.size(); + while (map.size() > MAX_SATELLITE_NODES) { + if (!evictStalestSatellite(*this, map)) + break; + } + if (map.size() != before) { + trimmedAny = true; + LOG_MIGRATION("Trimmed %s satellites %u -> %u (cap %d)", name, (unsigned)before, (unsigned)map.size(), + MAX_SATELLITE_NODES); + } + }; +#if !MESHTASTIC_EXCLUDE_POSITIONDB + trim(nodePositions, "position"); +#endif + +#if !MESHTASTIC_EXCLUDE_TELEMETRYDB + trim(nodeTelemetry, "telemetry"); +#endif + +#if !MESHTASTIC_EXCLUDE_ENVIRONMENTDB + trim(nodeEnvironment, "environment"); +#endif + +#if !MESHTASTIC_EXCLUDE_STATUSDB + trim(nodeStatus, "status"); +#endif + + (void)trim; // all four maps may be compiled out + return trimmedAny; +} + void NodeDB::cleanupMeshDB() { int newPos = 0, removed = 0; for (int i = 0; i < numMeshNodes; i++) { meshtastic_NodeInfoLite &n = meshNodes->at(i); - if (nodeInfoLiteHasUser(&n)) { + // Keep ignored (blocked) nodes even without user info: a block set by + // bare node ID has no NodeInfo and would otherwise be purged here, + // silently dropping the block. + if (nodeInfoLiteHasUser(&n) || nodeInfoLiteIsIgnored(&n)) { if (n.public_key.size > 0) { if (memfll(n.public_key.bytes, 0, n.public_key.size)) { n.public_key.size = 0; @@ -1486,8 +1634,16 @@ void NodeDB::cleanupMeshDB() } else { // No user info - drop this node and its satellites const NodeNum gone = n.num; - if (gone) + if (gone) { +#if WARM_NODE_COUNT > 0 + // Keep any key we learned (e.g. via a DM before the NodeInfo + // exchange completed) rather than losing it with the purge. + if (n.public_key.size == 32) + warmStore.absorb(gone, n.last_heard, n.public_key.bytes); +#endif + eraseNodeSatellites(gone); + } removed++; } } @@ -1518,12 +1674,15 @@ void NodeDB::installDefaultDeviceState() #else snprintf(owner.long_name, sizeof(owner.long_name), "Meshtastic %04x", getNodeNum() & 0x0ffff); #endif + clampLongName(owner.long_name); // vendor userprefs may exceed the local cap + #ifdef USERPREFS_CONFIG_OWNER_SHORT_NAME snprintf(owner.short_name, sizeof(owner.short_name), (const char *)USERPREFS_CONFIG_OWNER_SHORT_NAME); #else snprintf(owner.short_name, sizeof(owner.short_name), "%04x", getNodeNum() & 0x0ffff); #endif + snprintf(owner.id, sizeof(owner.id), "!%08x", getNodeNum()); // Default node ID now based on nodenum memcpy(owner.macaddr, ourMacAddr, sizeof(owner.macaddr)); owner.has_is_unmessagable = true; @@ -1637,6 +1796,114 @@ LoadFileResult NodeDB::loadProto(const char *filename, size_t protoSize, size_t return state; } +#if WARM_NODE_COUNT > 0 +void NodeDB::demoteOldestHotNodesToWarm() +{ + const int keep = MAX_NUM_NODES; + if (numMeshNodes <= keep) + return; + + // Protected nodes (favorite/ignored/verified) outrank recency and are demoted + // only when the store is full of them; within a class, most-recently-heard + // wins. Index 0 is self and stays put (sort from +1), as in runtime eviction. + std::sort(meshNodes->begin() + 1, meshNodes->begin() + numMeshNodes, + [](const meshtastic_NodeInfoLite &a, const meshtastic_NodeInfoLite &b) { + const bool ka = nodeInfoLiteIsProtected(&a); + const bool kb = nodeInfoLiteIsProtected(&b); + if (ka != kb) + return ka; + return a.last_heard > b.last_heard; + }); + + int demoted = 0; + for (int i = keep; i < numMeshNodes; i++) { + const meshtastic_NodeInfoLite &n = (*meshNodes)[i]; + if (n.num == 0) + continue; + // Keep the public key if we have one (40 B warm record); keyless nodes + // still get a placeholder so re-admission restores last_heard. + warmStore.absorb(n.num, n.last_heard, n.public_key.size > 0 ? n.public_key.bytes : nullptr); + // Demotion drops the node from the header table, so drop its satellites + // too (the eviction chokepoint) — they'd otherwise orphan until the next + // enforceSatelliteCaps pass. + eraseNodeSatellites(n.num); + demoted++; + } + numMeshNodes = keep; // the resize() in loadFromDisk reclaims the demoted tail + LOG_MIGRATION("NodeDB migration: demoted %d node(s) over %d into the warm tier (keepers preferred)", demoted, keep); +} +#endif + +void NodeDB::nodeDBSelfCare() +{ + if (!meshNodes) + return; + + const NodeNum self = getNodeNum(); + const bool nodesOverCap = numMeshNodes > MAX_NUM_NODES; + + // Confirm self is present and its key matches what we just (re)derived. A + // non-empty DB that doesn't contain us means a foreign/over-cap or corrupt + // nodes.proto was loaded; an empty DB is just a fresh device (no warning). + meshtastic_NodeInfoLite *selfNode = getMeshNode(self); + if (!selfNode && numMeshNodes > 0) { + LOG_WARN("NodeDB self-care: self 0x%08x absent from DB, re-adding", (unsigned)self); + } else if (selfNode && owner.public_key.size == 32 && selfNode->public_key.size == 32 && + memcmp(selfNode->public_key.bytes, owner.public_key.bytes, 32) != 0) { + LOG_WARN("NodeDB self-care: self 0x%08x key mismatch, refreshing", (unsigned)self); + } + + // Maintenance that must never touch self. Pin self to index 0 first so + // the positional demote/eviction scans (which skip index 0) provably exclude + // us, wherever the loaded file happened to place our row. + if (selfNode && numMeshNodes > 0 && selfNode != &meshNodes->at(0)) { + std::swap(meshNodes->at(0), *selfNode); + } + +#if WARM_NODE_COUNT > 0 + if (nodesOverCap) + demoteOldestHotNodesToWarm(); // demotes oldest NON-self overflow; index 0 (us) left in place +#endif + + if (numMeshNodes > MAX_NUM_NODES) { + LOG_WARN("NodeDB self-care: %d over cap %d, truncating", numMeshNodes, MAX_NUM_NODES); + numMeshNodes = MAX_NUM_NODES; + } + // Normalise the backing store to the hot cap so getOrCreateMeshNode always + // has spare slots to append into (it indexes meshNodes->at(numMeshNodes++)). + meshNodes->resize(MAX_NUM_NODES); + + const bool satsTrimmed = enforceSatelliteCaps(); + + // Ensure self exists, sits at index 0, and carries current owner info — after + // any demotion has freed a slot. Covers the foreign/fixture case where the + // loaded file did not contain us at all. + meshtastic_NodeInfoLite *info = getOrCreateMeshNode(self); + if (info) { + TypeConversions::CopyUserToNodeInfoLite(info, owner); + if (info != &meshNodes->at(0)) + std::swap(meshNodes->at(0), *info); + } + + // One-shot rewrite: only when we healed something, and never while storage + // is locked — a locked boot loads placeholder defaults that must not be written + // over the encrypted store; reloadFromDisk() re-runs self-care once unlocked. +#ifdef MESHTASTIC_ENCRYPTED_STORAGE + const bool storageLocked = EncryptedStorage::isLockdownActive() && !EncryptedStorage::isUnlocked(); +#else + const bool storageLocked = false; +#endif + + if ((nodesOverCap || satsTrimmed) && !storageLocked) { + LOG_MIGRATION("NodeDB self-care: healed store (nodes-over-cap:%s sats-trimmed:%s); rewriting nodes.proto once", + nodesOverCap ? "yes" : "no", satsTrimmed ? "yes" : "no"); + saveNodeDatabaseToDisk(); +#if WARM_NODE_COUNT > 0 + warmStore.saveIfDirty(); +#endif + } +} + void NodeDB::loadFromDisk() { // Mark the current device state as completely unusable, so that if we fail reading the entire file from @@ -1650,6 +1917,8 @@ void NodeDB::loadFromDisk() storageCorruptThisLoad = false; #endif + migrationSavePending = false; + meshtastic_Config_SecurityConfig backupSecurity = meshtastic_Config_SecurityConfig_init_zero; #ifdef ARCH_ESP32 @@ -1659,6 +1928,7 @@ void NodeDB::loadFromDisk() rmDir("/static/static"); // Remove bad static web files bundle from initial 2.5.13 release spiLock->unlock(); #endif + #ifdef FSCom #if defined(FACTORY_INSTALL) && !defined(ARCH_PORTDUINO) spiLock->lock(); @@ -1673,7 +1943,7 @@ void NodeDB::loadFromDisk() } } spiLock->unlock(); -#endif +#endif // FACTORY_INSTALL, not PORTDUINO spiLock->lock(); if (FSCom.exists(legacyPrefFileName)) { spiLock->unlock(); @@ -1691,7 +1961,8 @@ void NodeDB::loadFromDisk() spiLock->unlock(); } -#endif +#endif // FSCom + #ifdef MESHTASTIC_ENCRYPTED_STORAGE // Only take the locked-boot defaults path when lockdown is ACTIVE (the // device is provisioned) AND storage is still locked. A lockdown-capable @@ -1745,7 +2016,7 @@ void NodeDB::loadFromDisk() installDefaultNodeDatabase(); } else if (nodeDatabase.version < DEVICESTATE_CUR_VER) { if (migrateLegacyNodeDatabase()) - saveNodeDatabaseToDisk(); + migrationSavePending = true; else installDefaultNodeDatabase(); } else { @@ -1758,33 +2029,40 @@ void NodeDB::loadFromDisk() #else 0u; #endif + const unsigned telCount = #if !MESHTASTIC_EXCLUDE_TELEMETRYDB (unsigned)nodeTelemetry.size(); #else 0u; #endif + const unsigned envCount = #if !MESHTASTIC_EXCLUDE_ENVIRONMENTDB (unsigned)nodeEnvironment.size(); #else 0u; #endif + const unsigned statusCount = #if !MESHTASTIC_EXCLUDE_STATUSDB (unsigned)nodeStatus.size(); #else 0u; #endif + LOG_INFO("Loaded saved nodedatabase v%d: %d nodes, %u pos, %u tel, %u env, %u status", nodeDatabase.version, nodeDatabase.nodes.size(), posCount, telCount, envCount, statusCount); } - if (numMeshNodes > MAX_NUM_NODES) { - LOG_WARN("Node count %d exceeds MAX_NUM_NODES %d, truncating", numMeshNodes, MAX_NUM_NODES); - numMeshNodes = MAX_NUM_NODES; - } - meshNodes->resize(MAX_NUM_NODES); + // Left UNTRIMMED on purpose: trim/demote/satellite-cap/self-pin/rewrite all + // run in nodeDBSelfCare() once getNodeNum() is valid (still 0 here on a cold + // boot, so we could only assume index 0 == self — the very bug being fixed). +#if WARM_NODE_COUNT > 0 + // Load the warm tier so its on-disk snapshot is available before the node DB + // is exercised (and before nodeDBSelfCare() demotes any overflow into it). + warmStore.load(); +#endif // static DeviceState scratch; We no longer read into a tempbuf because this structure is 15KB of valuable RAM state = loadProto(deviceStateFileName, meshtastic_DeviceState_size, sizeof(meshtastic_DeviceState), @@ -1856,18 +2134,23 @@ void NodeDB::loadFromDisk() #ifdef USERPREFS_CONFIG_LORA_REGION config.lora.region = USERPREFS_CONFIG_LORA_REGION; #endif + #ifdef USERPREFS_LORACONFIG_USE_PRESET config.lora.use_preset = USERPREFS_LORACONFIG_USE_PRESET; #endif + #ifdef USERPREFS_LORACONFIG_BANDWIDTH config.lora.bandwidth = USERPREFS_LORACONFIG_BANDWIDTH; #endif + #ifdef USERPREFS_LORACONFIG_SPREAD_FACTOR config.lora.spread_factor = USERPREFS_LORACONFIG_SPREAD_FACTOR; #endif + #ifdef USERPREFS_LORACONFIG_CODING_RATE config.lora.coding_rate = USERPREFS_LORACONFIG_CODING_RATE; #endif + #ifdef USERPREFS_LORACONFIG_OVERRIDE_FREQUENCY config.lora.override_frequency = USERPREFS_LORACONFIG_OVERRIDE_FREQUENCY; #endif @@ -1884,6 +2167,7 @@ void NodeDB::loadFromDisk() #if defined(USERPREFS_USE_ADMIN_KEY_0) || defined(USERPREFS_USE_ADMIN_KEY_1) || defined(USERPREFS_USE_ADMIN_KEY_2) uint16_t sum = 0; #endif + #ifdef USERPREFS_USE_ADMIN_KEY_0 for (uint8_t b = 0; b < 32; b++) { @@ -2076,6 +2360,17 @@ bool NodeDB::reloadFromDisk() return false; } + // loadFromDisk() leaves the store untrimmed; run self-care now (getNodeNum() + // is valid at runtime) to trim/demote non-self overflow, pin self to index 0 + // and normalise the backing store before the node DB is exercised again. + nodeDBSelfCare(); + + // Preserve constructor ordering: persist any migration only after self-care. + if (migrationSavePending) { + saveNodeDatabaseToDisk(); + migrationSavePending = false; + } + // Push the now-real config to the radio. if (rIface) { channels.onConfigChanged(); @@ -2196,6 +2491,7 @@ bool NodeDB::saveChannelsToDisk() FSCom.mkdir("/prefs"); spiLock->unlock(); #endif + return saveProto(channelFileName, meshtastic_ChannelFile_size, &meshtastic_ChannelFile_msg, &channelFile); } @@ -2251,6 +2547,7 @@ bool NodeDB::saveNodeDatabaseToDisk() FSCom.mkdir("/prefs"); spiLock->unlock(); #endif + // Project the maps into the on-disk vectors just before encoding; cleared // again on the way out so we don't carry duplicate state. concurrency::LockGuard guard(&satelliteMutex); @@ -2267,6 +2564,7 @@ bool NodeDB::saveNodeDatabaseToDisk() #else nodeDatabase.positions.clear(); #endif + #if !MESHTASTIC_EXCLUDE_TELEMETRYDB nodeDatabase.telemetry.clear(); nodeDatabase.telemetry.reserve(nodeTelemetry.size()); @@ -2280,6 +2578,7 @@ bool NodeDB::saveNodeDatabaseToDisk() #else nodeDatabase.telemetry.clear(); #endif + #if !MESHTASTIC_EXCLUDE_ENVIRONMENTDB nodeDatabase.environment.clear(); nodeDatabase.environment.reserve(nodeEnvironment.size()); @@ -2293,6 +2592,7 @@ bool NodeDB::saveNodeDatabaseToDisk() #else nodeDatabase.environment.clear(); #endif + #if !MESHTASTIC_EXCLUDE_STATUSDB nodeDatabase.status.clear(); nodeDatabase.status.reserve(nodeStatus.size()); @@ -2319,6 +2619,11 @@ bool NodeDB::saveNodeDatabaseToDisk() nodeDatabase.environment.shrink_to_fit(); nodeDatabase.status.clear(); nodeDatabase.status.shrink_to_fit(); +#if WARM_NODE_COUNT > 0 + // Same cadence as the node DB; failure is logged but must not propagate — + // a false return from here would trigger saveToDisk()'s fsFormat() path. + warmStore.saveIfDirty(); +#endif return ok; } @@ -2354,6 +2659,7 @@ bool NodeDB::saveToDiskNoRetry(int saveWhat) FSCom.mkdir("/prefs"); spiLock->unlock(); #endif + if (saveWhat & SEGMENT_CONFIG) { config.has_device = true; config.has_display = true; @@ -2555,6 +2861,7 @@ void NodeDB::updatePosition(uint32_t nodeId, const meshtastic_Position &p, RxSou #else { concurrency::LockGuard guard(&satelliteMutex); + evictSatelliteOverCap(*this, nodePositions, nodeId); meshtastic_PositionLite &slot = nodePositions[nodeId]; // creates default-zero entry if missing if (src == RX_SRC_LOCAL) { @@ -2612,8 +2919,10 @@ void NodeDB::updateTelemetry(uint32_t nodeId, const meshtastic_Telemetry &t, RxS } #if !MESHTASTIC_EXCLUDE_TELEMETRYDB concurrency::LockGuard guard(&satelliteMutex); + evictSatelliteOverCap(*this, nodeTelemetry, nodeId); nodeTelemetry[nodeId] = t.variant.device_metrics; #endif + } else if (t.which_variant == meshtastic_Telemetry_environment_metrics_tag) { if (src == RX_SRC_LOCAL) { LOG_DEBUG("updateTelemetry LOCAL env"); @@ -2622,8 +2931,10 @@ void NodeDB::updateTelemetry(uint32_t nodeId, const meshtastic_Telemetry &t, RxS } #if !MESHTASTIC_EXCLUDE_ENVIRONMENTDB concurrency::LockGuard guard(&satelliteMutex); + evictSatelliteOverCap(*this, nodeEnvironment, nodeId); nodeEnvironment[nodeId] = t.variant.environment_metrics; #endif + } else { return; // air_quality / power / local_stats / health: not stored per-node } @@ -2652,13 +2963,13 @@ void NodeDB::addFromContact(meshtastic_SharedContact contact) info->num = contact.node_num; TypeConversions::CopyUserToNodeInfoLite(info, contact.user); if (contact.should_ignore) { - // If should_ignore is set, - // we need to clear the public key and other cruft, in addition to setting the node as ignored - nodeInfoLiteSetBit(info, NODEINFO_BITFIELD_IS_IGNORED_MASK, true); + // Block the contact and drop its rich satellite data, but keep the + // public key copied above — an ignored peer keeps a usable identity + // (a verifiable target) rather than a bare node number. + if (!setProtectedFlag(info, NODEINFO_BITFIELD_IS_IGNORED_MASK, true)) + LOG_WARN(PROTECTED_CAP_WARN_FMT, "ignore", contact.node_num, MAX_NUM_NODES - 2); nodeInfoLiteSetBit(info, NODEINFO_BITFIELD_IS_FAVORITE_MASK, false); eraseNodeSatellites(contact.node_num); - info->public_key.size = 0; - memset(info->public_key.bytes, 0, sizeof(info->public_key.bytes)); } else { /* Clients are sending add_contact before every text message DM (because clients may hold a larger node database with * public keys than the radio holds). However, we don't want to update last_heard just because we sent someone a DM! @@ -2677,12 +2988,16 @@ void NodeDB::addFromContact(meshtastic_SharedContact contact) } else { // Normal case: set is_favorite to prevent expiration. // last_heard will remain as-is (or remain 0 if this entry wasn't in the nodeDB). - nodeInfoLiteSetBit(info, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true); + // If the protected cap refuses the favorite, fall back to stamping last_heard so the + // contact still isn't the first eviction victim. + if (!setProtectedFlag(info, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true)) + info->last_heard = getTime(); } // As the clients will begin sending the contact with DMs, we want to strictly check if the node is manually verified if (contact.manually_verified) { - nodeInfoLiteSetBit(info, NODEINFO_BITFIELD_IS_KEY_MANUALLY_VERIFIED_MASK, true); + if (!setProtectedFlag(info, NODEINFO_BITFIELD_IS_KEY_MANUALLY_VERIFIED_MASK, true)) + LOG_WARN(PROTECTED_CAP_WARN_FMT, "verify", contact.node_num, MAX_NUM_NODES - 2); } // Mark the node's key as manually verified to indicate trustworthiness. updateGUIforNode = info; @@ -2815,14 +3130,48 @@ void NodeDB::updateFrom(const meshtastic_MeshPacket &mp) } } -void NodeDB::set_favorite(bool is_favorite, uint32_t nodeId) +int NodeDB::numProtectedNodes() const +{ + int count = 0; + for (int i = 0; i < numMeshNodes; i++) + if (nodeInfoLiteIsProtected(&meshNodes->at(i))) + count++; + return count; +} + +bool NodeDB::setProtectedFlag(meshtastic_NodeInfoLite *node, uint32_t mask, bool on) +{ + if (!node) + return false; + if (!on) { + nodeInfoLiteSetBit(node, mask, false); + return true; + } + // Adding a flag to a node that is already protected doesn't grow the + // protected set, so it's always allowed. A newly-protected node is refused + // once the protected set has reached MAX_NUM_NODES-2, leaving two evictable + // slots so getOrCreateMeshNode can always make room. + if (nodeInfoLiteIsProtected(node) || numProtectedNodes() < MAX_NUM_NODES - 2) { + nodeInfoLiteSetBit(node, mask, true); + return true; + } + return false; +} + +bool NodeDB::set_favorite(bool is_favorite, uint32_t nodeId) { meshtastic_NodeInfoLite *lite = getMeshNode(nodeId); - if (lite && nodeInfoLiteIsFavorite(lite) != is_favorite) { - nodeInfoLiteSetBit(lite, NODEINFO_BITFIELD_IS_FAVORITE_MASK, is_favorite); + if (!lite) + return false; + if (nodeInfoLiteIsFavorite(lite) == is_favorite) + return true; // already in the requested state + if (setProtectedFlag(lite, NODEINFO_BITFIELD_IS_FAVORITE_MASK, is_favorite)) { sortMeshDB(); saveNodeDatabaseToDisk(); + return true; } + LOG_WARN(PROTECTED_CAP_WARN_FMT, "favorite", nodeId, MAX_NUM_NODES - 2); + return false; } bool NodeDB::isFavorite(uint32_t nodeId) @@ -2952,6 +3301,30 @@ bool NodeDB::isFull() return (numMeshNodes >= MAX_NUM_NODES) || (memGet.getFreeHeap() < MINIMUM_SAFE_FREE_HEAP); } +uint32_t NodeDB::hotNodeLastHeard(NodeNum n) const +{ + for (int i = 0; i < numMeshNodes; i++) + if (meshNodes->at(i).num == n) + return meshNodes->at(i).last_heard; + return 0; +} + +bool NodeDB::copyPublicKey(NodeNum n, meshtastic_NodeInfoLite_public_key_t &out) +{ + const meshtastic_NodeInfoLite *info = getMeshNode(n); + if (info && info->public_key.size == 32) { + out = info->public_key; + return true; + } +#if WARM_NODE_COUNT > 0 + if (warmStore.copyKey(n, out.bytes)) { + out.size = 32; + return true; + } +#endif + return false; +} + /// Find a node in our DB, create an empty NodeInfo if missing meshtastic_NodeInfoLite *NodeDB::getOrCreateMeshNode(NodeNum n) { @@ -2988,7 +3361,14 @@ meshtastic_NodeInfoLite *NodeDB::getOrCreateMeshNode(NodeNum n) } if (oldestIndex != -1) { - eraseNodeSatellites(meshNodes->at(oldestIndex).num); + const meshtastic_NodeInfoLite &evicted = meshNodes->at(oldestIndex); +#if WARM_NODE_COUNT > 0 + // Demote to the warm tier so the identity (and crucially the + // PKI key) outlives the hot-store slot. + warmStore.absorb(evicted.num, evicted.last_heard, + evicted.public_key.size == 32 ? evicted.public_key.bytes : NULL); +#endif + eraseNodeSatellites(evicted.num); // Shove the remaining nodes down the chain for (int i = oldestIndex; i < numMeshNodes - 1; i++) { meshNodes->at(i) = meshNodes->at(i + 1); @@ -2996,12 +3376,33 @@ meshtastic_NodeInfoLite *NodeDB::getOrCreateMeshNode(NodeNum n) (numMeshNodes)--; } } + // Don't append past the end of the vector. The protected-node cap + // (numProtectedNodes() <= MAX_NUM_NODES-2) means the eviction above frees + // a slot in normal operation; this guards the legacy case of a pre-cap + // database that is full of protected nodes — refuse rather than overrun. + if (numMeshNodes >= MAX_NUM_NODES) + return NULL; + // Pre-size before append when run before nodeDBSelfCare() (boot keygen); else at() aborts on nRF52. + if (static_cast(numMeshNodes) >= meshNodes->size()) + meshNodes->resize(numMeshNodes + 1); // add the node at the end lite = &meshNodes->at((numMeshNodes)++); // everything is missing except the nodenum memset(lite, 0, sizeof(*lite)); lite->num = n; +#if WARM_NODE_COUNT > 0 + // Re-admission: restore what the warm tier kept for this node + WarmNodeEntry warm; + if (warmStore.take(n, warm)) { + lite->last_heard = warm.last_heard; + if (!memfll(warm.public_key, 0, sizeof(warm.public_key))) { + lite->public_key.size = 32; + memcpy(lite->public_key.bytes, warm.public_key, 32); + } + LOG_MIGRATION("Rehydrated node 0x%x from warm tier (key=%d)", n, lite->public_key.size == 32); + } +#endif LOG_INFO("Adding node to database with %i nodes and %u bytes free!", numMeshNodes, memGet.getFreeHeap()); } diff --git a/src/mesh/NodeDB.h b/src/mesh/NodeDB.h index 3fe244294..f2df5ef1d 100644 --- a/src/mesh/NodeDB.h +++ b/src/mesh/NodeDB.h @@ -11,6 +11,7 @@ #include "MeshTypes.h" #include "NodeStatus.h" +#include "WarmNodeStore.h" #include "concurrency/Lock.h" #include "configuration.h" #include "mesh-pb-constants.h" @@ -226,9 +227,25 @@ class NodeDB bool updateUser(uint32_t nodeId, meshtastic_User &p, uint8_t channelIndex = 0); /* - * Sets a node either favorite or unfavorite + * Sets a node either favorite or unfavorite. Returns true if the node ends + * up in the requested state; false if the node is unknown or favouriting + * was refused by the protected-node cap (MAX_NUM_NODES - 2). */ - void set_favorite(bool is_favorite, uint32_t nodeId); + bool set_favorite(bool is_favorite, uint32_t nodeId); + + /// Count of eviction-protected (favourite/ignored/manually-verified) nodes. + int numProtectedNodes() const; + + /// printf-style warning emitted when setProtectedFlag() refuses a node at + /// the cap. %s = verb (favorite/ignore), 0x%08x = node, %d = cap. Shared by + /// LOG_WARN here and AdminModule::sendWarning so the wording stays in sync. + static constexpr const char *PROTECTED_CAP_WARN_FMT = "Can't %s 0x%08x: protected-node limit (%d) reached"; + + /// Turn an eviction-protection flag (favourite/ignored/verified) on/off. Off + /// always succeeds; on returns false (no change) once the protected set hits + /// the cap (MAX_NUM_NODES-2), keeping >=2 always-evictable slots. Callers + /// surface the refusal to the user. + bool setProtectedFlag(meshtastic_NodeInfoLite *node, uint32_t mask, bool on); /* * Returns true if the node is in the NodeDB and marked as favorite @@ -295,6 +312,24 @@ class NodeDB virtual meshtastic_NodeInfoLite *getMeshNode(NodeNum n); size_t getNumMeshNodes() { return numMeshNodes; } + /// Find a node in our DB, create an empty NodeInfoLite if missing (evicting + /// the oldest non-protected node when full). Public so admin handlers can + /// register a node we have not heard from yet (e.g. to block it by ID). + meshtastic_NodeInfoLite *getOrCreateMeshNode(NodeNum n); + +#if WARM_NODE_COUNT > 0 + // Warm ("long-tail") tier: minimal {num, last_heard, public_key} records + // for nodes evicted from the hot store. See WarmNodeStore.h. + WarmNodeStore warmStore; +#endif + + /// Copy the 32-byte public key for node n — hot store first, then the warm + /// tier. Returns false if we don't know a key for n. + bool copyPublicKey(NodeNum n, meshtastic_NodeInfoLite_public_key_t &out); + + /// last_heard of a hot-store node, or 0 if absent. Plain scan of meshNodes + /// with no allocation side effects (unlike getOrCreateMeshNode). + uint32_t hotNodeLastHeard(NodeNum n) const; // Thread-safe satellite-map accessors. Return false if absent or the // corresponding DB is compiled out. @@ -341,11 +376,15 @@ class NodeDB emptyNodeDatabase.version = DEVICESTATE_CUR_VER; size_t nodeDatabaseSize; pb_get_encoded_size(&nodeDatabaseSize, meshtastic_NodeDatabase_fields, &emptyNodeDatabase); - // Always include satellite slots so backups from higher-cap peers - // decode without truncation, even when our build excludes the DBs. - return nodeDatabaseSize + (MAX_NUM_NODES * meshtastic_NodeInfoLite_size) + - (MAX_NUM_NODES * meshtastic_NodePositionEntry_size) + (MAX_NUM_NODES * meshtastic_NodeTelemetryEntry_size) + - (MAX_NUM_NODES * meshtastic_NodeEnvironmentEntry_size) + (MAX_NUM_NODES * meshtastic_NodeStatusEntry_size); + // Decode-stream size ceiling only — no buffer this big is allocated (load + // streams from the file). Sized for the largest file any prior firmware + // could write (250-node ESP32-S3, satellites uncapped) so capacity + // downgrades / peer backups still decode; excess is trimmed after load. + // (not constexpr: portduino resolves MAX_NUM_NODES from runtime config) + const size_t loadCeiling = ((size_t)MAX_NUM_NODES > 250) ? (size_t)MAX_NUM_NODES : 250; + return nodeDatabaseSize + (loadCeiling * meshtastic_NodeInfoLite_size) + + (loadCeiling * meshtastic_NodePositionEntry_size) + (loadCeiling * meshtastic_NodeTelemetryEntry_size) + + (loadCeiling * meshtastic_NodeEnvironmentEntry_size) + (loadCeiling * meshtastic_NodeStatusEntry_size); } // returns true if the maximum number of nodes is reached or we are running low on memory @@ -430,11 +469,10 @@ class NodeDB mutable concurrency::Lock satelliteMutex; bool duplicateWarned = false; bool localPositionUpdatedSinceBoot = false; + bool migrationSavePending = false; uint32_t lastNodeDbSave = 0; // when we last saved our db to flash uint32_t lastBackupAttempt = 0; // when we last tried a backup automatically or manually uint32_t lastSort = 0; // When last sorted the nodeDB - /// Find a node in our DB, create an empty NodeInfoLite if missing - meshtastic_NodeInfoLite *getOrCreateMeshNode(NodeNum n); /* * Internal boolean to track sorting paused @@ -447,9 +485,33 @@ class NodeDB /// read our db from flash void loadFromDisk(); +#ifdef PIO_UNIT_TESTING + // Grant the unit-test shim access to the private maintenance paths below + // (migration / cleanup / eviction) without relaxing production access. + friend class NodeDBTestShim; +#endif + /// purge db entries without user info void cleanupMeshDB(); + /// Trim each satellite map down to MAX_SATELLITE_NODES, dropping the + /// stalest entries (used after loading files written before the cap, or by + /// a build with a larger cap). Returns true iff anything was trimmed. + bool enforceSatelliteCaps(); + + /// Node-DB self-care; call only once identity is established (getNodeNum() + /// valid). Confirms self is present, trims/demotes only NON-self overflow, and + /// rewrites the store once when something changed (never while storage locked). + void nodeDBSelfCare(); + +#if WARM_NODE_COUNT > 0 + /// A database from a larger-cap build (e.g. the pre-fork 150-node nRF52 store) + /// can exceed MAX_NUM_NODES on load. Rank the hot store, demote the oldest + /// overflow into the warm tier preserving {num, last_heard, public_key} so PKI + /// DMs survive instead of dropping on truncation. + void demoteOldestHotNodesToWarm(); +#endif + /// Reinit device state from scratch (not loading from disk) void installDefaultDeviceState(), installDefaultNodeDatabase(), installDefaultChannels(), installDefaultConfig(bool preserveKey), installDefaultModuleConfig(); @@ -570,6 +632,12 @@ inline bool nodeInfoLiteHasXeddsaSigned(const meshtastic_NodeInfoLite *n) { return n && (n->bitfield & NODEINFO_BITFIELD_HAS_XEDDSA_SIGNED_MASK); } +/// A node that the eviction/migration paths must not drop: a favourite, an +/// ignored (blocked) node, or a manually-verified key. +inline bool nodeInfoLiteIsProtected(const meshtastic_NodeInfoLite *n) +{ + return nodeInfoLiteIsFavorite(n) || nodeInfoLiteIsIgnored(n) || nodeInfoLiteIsKeyManuallyVerified(n); +} inline void nodeInfoLiteSetBit(meshtastic_NodeInfoLite *n, uint32_t mask, bool value) { diff --git a/src/mesh/NodeDBLegacyMigration.cpp b/src/mesh/NodeDBLegacyMigration.cpp index 18d6daa4d..107a4e32d 100644 --- a/src/mesh/NodeDBLegacyMigration.cpp +++ b/src/mesh/NodeDBLegacyMigration.cpp @@ -14,6 +14,7 @@ #include "configuration.h" #include "mesh-pb-constants.h" #include "mesh/generated/meshtastic/deviceonly_legacy.pb.h" +#include "meshUtils.h" #include #include @@ -88,8 +89,10 @@ bool NodeDB::migrateLegacyNodeDatabase() slim.bitfield |= NODEINFO_BITFIELD_HAS_USER_MASK; strncpy(slim.long_name, legacy.user.long_name, sizeof(slim.long_name)); slim.long_name[sizeof(slim.long_name) - 1] = '\0'; + sanitizeUtf8(slim.long_name, sizeof(slim.long_name)); // replace bad bytes so nanopb encode never fails strncpy(slim.short_name, legacy.user.short_name, sizeof(slim.short_name)); slim.short_name[sizeof(slim.short_name) - 1] = '\0'; + sanitizeUtf8(slim.short_name, sizeof(slim.short_name)); // same — v24 names may contain non-UTF-8 bytes slim.hw_model = legacy.user.hw_model; slim.role = legacy.user.role; if (legacy.user.is_licensed) diff --git a/src/mesh/Router.cpp b/src/mesh/Router.cpp index f176f60e6..9b3aa4676 100644 --- a/src/mesh/Router.cpp +++ b/src/mesh/Router.cpp @@ -485,14 +485,16 @@ DecodeState perhapsDecode(meshtastic_MeshPacket *p) bool decrypted = false; ChannelIndex chIndex = 0; #if !(MESHTASTIC_EXCLUDE_PKI) - // Attempt PKI decryption first - if (p->channel == 0 && isToUs(p) && p->to > 0 && !isBroadcast(p->to) && nodeDB->getMeshNode(p->from) != nullptr && - nodeDB->getMeshNode(p->from)->public_key.size > 0 && nodeDB->getMeshNode(p->to) != nullptr && - nodeDB->getMeshNode(p->to)->public_key.size > 0 && rawSize > MESHTASTIC_PKC_OVERHEAD) { + // Attempt PKI decryption first. The sender's key may come from the hot + // store or the warm tier (nodes evicted from the hot store keep their key + // there), so DMs from long-tail nodes still decrypt. + meshtastic_NodeInfoLite_public_key_t fromKey = {0, {0}}; + if (p->channel == 0 && isToUs(p) && p->to > 0 && !isBroadcast(p->to) && nodeDB->copyPublicKey(p->from, fromKey) && + nodeDB->getMeshNode(p->to) != nullptr && nodeDB->getMeshNode(p->to)->public_key.size > 0 && + rawSize > MESHTASTIC_PKC_OVERHEAD) { LOG_DEBUG("Attempt PKI decryption"); - if (crypto->decryptCurve25519(p->from, nodeDB->getMeshNode(p->from)->public_key, p->id, rawSize, p->encrypted.bytes, - bytes)) { + if (crypto->decryptCurve25519(p->from, fromKey, p->id, rawSize, p->encrypted.bytes, bytes)) { LOG_INFO("PKI Decryption worked!"); meshtastic_Data decodedtmp; @@ -503,7 +505,7 @@ DecodeState perhapsDecode(meshtastic_MeshPacket *p) decrypted = true; LOG_INFO("Packet decrypted using PKI!"); p->pki_encrypted = true; - memcpy(&p->public_key.bytes, nodeDB->getMeshNode(p->from)->public_key.bytes, 32); + memcpy(p->public_key.bytes, fromKey.bytes, 32); p->public_key.size = 32; p->decoded = decodedtmp; p->which_payload_variant = meshtastic_MeshPacket_decoded_tag; // change type to decoded @@ -720,7 +722,10 @@ meshtastic_Routing_Error perhapsEncode(meshtastic_MeshPacket *p) ChannelIndex chIndex = p->channel; // keep as a local because we are about to change it #if !(MESHTASTIC_EXCLUDE_PKI) - meshtastic_NodeInfoLite *node = nodeDB->getMeshNode(p->to); + // Destination key from the hot store or the warm tier (evicted + // long-tail nodes keep their key there) + meshtastic_NodeInfoLite_public_key_t destKey = {0, {0}}; + bool haveDestKey = nodeDB->copyPublicKey(p->to, destKey); // We may want to retool things so we can send a PKC packet when the client specifies a key and nodenum, even if the node // is not in the local nodedb // First, only PKC encrypt packets we are originating @@ -743,18 +748,17 @@ meshtastic_Routing_Error perhapsEncode(meshtastic_MeshPacket *p) if (numbytes + MESHTASTIC_HEADER_LENGTH + MESHTASTIC_PKC_OVERHEAD > MAX_LORA_PAYLOAD_LEN) return meshtastic_Routing_Error_TOO_LARGE; // Check for a known public key for the destination - if (node == nullptr || node->public_key.size != 32) { + if (!haveDestKey) { LOG_WARN("Unknown public key for destination node 0x%08x (portnum %d), refusing to send legacy DM", p->to, p->decoded.portnum); return meshtastic_Routing_Error_PKI_SEND_FAIL_PUBLIC_KEY; } - if (p->pki_encrypted && !memfll(p->public_key.bytes, 0, 32) && - memcmp(p->public_key.bytes, node->public_key.bytes, 32) != 0) { + if (p->pki_encrypted && !memfll(p->public_key.bytes, 0, 32) && memcmp(p->public_key.bytes, destKey.bytes, 32) != 0) { LOG_WARN("Client public key differs from requested: 0x%02x, stored key begins 0x%02x", *p->public_key.bytes, - *node->public_key.bytes); + *destKey.bytes); return meshtastic_Routing_Error_PKI_FAILED; } - crypto->encryptCurve25519(p->to, getFrom(p), node->public_key, p->id, numbytes, bytes, p->encrypted.bytes); + crypto->encryptCurve25519(p->to, getFrom(p), destKey, p->id, numbytes, bytes, p->encrypted.bytes); numbytes += MESHTASTIC_PKC_OVERHEAD; p->channel = 0; p->pki_encrypted = true; diff --git a/src/mesh/WarmNodeStore.cpp b/src/mesh/WarmNodeStore.cpp new file mode 100644 index 000000000..8617f505c --- /dev/null +++ b/src/mesh/WarmNodeStore.cpp @@ -0,0 +1,548 @@ +#include "WarmNodeStore.h" + +#if WARM_NODE_COUNT > 0 + +#include "FSCommon.h" +#include "SPILock.h" +#include "SafeFile.h" +#include "configuration.h" +#include "power/PowerHAL.h" +#include +#include + +#if defined(NRF52840_XXAA) +#include "flash/flash_nrf5x.h" +#define WARM_RING_MAGIC 0x474E5257u // "WRNG" +// A tombstone is an entry record whose last_heard is all-ones — getTime() +// (unix seconds) cannot reach 0xFFFFFFFF until 2106, and erased flash is +// detected via num == 0xFFFFFFFF before last_heard is ever inspected. +#define WARM_RING_TOMBSTONE 0xFFFFFFFFu +#else +// warm.dat layout: this header followed by count packed WarmNodeEntry records. +struct WarmStoreHeader { + uint32_t magic; // WARM_STORE_MAGIC + uint32_t reserved; // 0; kept so the header stays 16 B + uint16_t count; // entries persisted + uint16_t entrySize; // sizeof(WarmNodeEntry), format guard + uint32_t crc; // crc32 over count * entrySize bytes +}; +static_assert(sizeof(WarmStoreHeader) == 16, "header layout is part of the persistence format"); + +#define WARM_STORE_MAGIC 0x314D5257u // "WRM1" + +#ifdef FSCom +static const char *warmFileName = "/prefs/warm.dat"; +#endif +#endif // NRF52840_XXAA + +static inline bool keyIsSet(const uint8_t key[32]) +{ + for (int i = 0; i < 32; i++) + if (key[i]) + return true; + return false; +} + +WarmNodeStore::WarmNodeStore() +{ +#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) + entries = static_cast(ps_calloc(WARM_NODE_COUNT, sizeof(WarmNodeEntry))); + if (!entries) { + LOG_WARN("WarmStore: PSRAM alloc failed, using heap"); + entries = static_cast(calloc(WARM_NODE_COUNT, sizeof(WarmNodeEntry))); + } +#else + entries = static_cast(calloc(WARM_NODE_COUNT, sizeof(WarmNodeEntry))); +#endif +#if defined(NRF52840_XXAA) + memset(pageOf, kNoPage, sizeof(pageOf)); +#endif +} + +WarmNodeStore::~WarmNodeStore() +{ + free(entries); // always malloc-family (calloc / ps_calloc) + entries = nullptr; +} + +WarmNodeEntry *WarmNodeStore::find(NodeNum num) const +{ + if (!entries || !num) + return nullptr; + for (size_t i = 0; i < WARM_NODE_COUNT; i++) + if (entries[i].num == num) + return &entries[i]; + return nullptr; +} + +// Slot placement with the keyed-first admission policy. Shared by absorb() +// and the ring replay, so the policy is applied identically in both paths. +WarmNodeEntry *WarmNodeStore::place(NodeNum num, uint32_t lastHeard, const uint8_t *key32) +{ + if (!entries || !num) + return nullptr; + + const bool candidateKeyed = key32 && keyIsSet(key32); + + WarmNodeEntry *slot = find(num); + const bool sameNode = slot != nullptr; + if (!slot) { + // Pick a victim: any empty slot, else the oldest keyless entry, else + // (only for keyed candidates) the oldest keyed entry. + WarmNodeEntry *oldestKeyless = nullptr, *oldestKeyed = nullptr; + for (size_t i = 0; i < WARM_NODE_COUNT; i++) { + WarmNodeEntry &e = entries[i]; + if (!e.num) { + slot = &e; + break; + } + if (keyIsSet(e.public_key)) { + if (!oldestKeyed || e.last_heard < oldestKeyed->last_heard) + oldestKeyed = &e; + } else { + if (!oldestKeyless || e.last_heard < oldestKeyless->last_heard) + oldestKeyless = &e; + } + } + if (!slot) + slot = oldestKeyless ? oldestKeyless : (candidateKeyed ? oldestKeyed : nullptr); + if (!slot) + return nullptr; // store full of keyed entries and the candidate has no key + } + + slot->num = num; + slot->last_heard = lastHeard; + if (candidateKeyed) + memcpy(slot->public_key, key32, 32); + else if (!sameNode) + // Repurposing a victim slot for a different node: clear its stale key. + // A keyless refresh of a node already here keeps the key we learned. + memset(slot->public_key, 0, 32); + return slot; +} + +bool WarmNodeStore::absorb(NodeNum num, uint32_t lastHeard, const uint8_t *key32) +{ + const WarmNodeEntry *slot = place(num, lastHeard, key32); + if (!slot) + return false; + persistEntry(*slot); + LOG_MIGRATION("WarmStore absorb 0x%08x key=%d last_heard=%u (now %u/%u)", (unsigned)num, keyIsSet(slot->public_key) ? 1 : 0, + (unsigned)lastHeard, (unsigned)count(), (unsigned)capacity()); + return true; +} + +bool WarmNodeStore::take(NodeNum num, WarmNodeEntry &out) +{ + WarmNodeEntry *e = find(num); + if (!e) + return false; + out = *e; + const int idx = static_cast(e - entries); + memset(e, 0, sizeof(*e)); + persistRemove(num, idx); + LOG_MIGRATION("WarmStore take(rehydrate) 0x%08x key=%d (now %u/%u)", (unsigned)num, keyIsSet(out.public_key) ? 1 : 0, + (unsigned)count(), (unsigned)capacity()); + return true; +} + +#if MESHTASTIC_NODEDB_MIGRATION_VERBOSE +void WarmNodeStore::dumpToLog(const char *reason) const +{ + if (!entries) { + LOG_MIGRATION("WarmStore dump (%s): backend not allocated", reason); + return; + } + LOG_MIGRATION("WarmStore dump (%s): %u live / %u cap ==>", reason, (unsigned)count(), (unsigned)capacity()); + unsigned shown = 0; + for (size_t i = 0; i < WARM_NODE_COUNT; i++) { + const WarmNodeEntry &e = entries[i]; + if (e.num == 0) + continue; + LOG_MIGRATION(" warm[%3u] 0x%08x last_heard=%u key=%d", (unsigned)i, (unsigned)e.num, (unsigned)e.last_heard, + keyIsSet(e.public_key) ? 1 : 0); + shown++; + } + LOG_MIGRATION("WarmStore dump (%s): <== end (%u entries)", reason, shown); +} +#endif // MESHTASTIC_NODEDB_MIGRATION_VERBOSE + +bool WarmNodeStore::copyKey(NodeNum num, uint8_t out[32]) const +{ + const WarmNodeEntry *e = find(num); + if (!e || !keyIsSet(e->public_key)) + return false; + memcpy(out, e->public_key, 32); + return true; +} + +bool WarmNodeStore::contains(NodeNum num) const +{ + return find(num) != nullptr; +} + +void WarmNodeStore::remove(NodeNum num) +{ + WarmNodeEntry *e = find(num); + if (e) { + const int idx = static_cast(e - entries); + memset(e, 0, sizeof(*e)); + persistRemove(num, idx); + } +} + +void WarmNodeStore::clear() +{ + if (!entries) + return; + memset(entries, 0, WARM_NODE_COUNT * sizeof(WarmNodeEntry)); +#if defined(NRF52840_XXAA) + memset(pageOf, kNoPage, sizeof(pageOf)); +#endif + persistClear(); +} + +size_t WarmNodeStore::count() const +{ + size_t n = 0; + if (entries) + for (size_t i = 0; i < WARM_NODE_COUNT; i++) + if (entries[i].num) + n++; + return n; +} + +bool WarmNodeStore::saveIfDirty() +{ + if (!dirty) + return true; + bool ok = save(); + if (ok) + dirty = false; + return ok; +} + +#if defined(NRF52840_XXAA) + +// Raw-flash record-ring backend (nRF52840). +// 3 × 4 KB pages below LittleFS. Mutations append 40 B records (entry snapshot, +// or tombstone with last_heard == 0xFFFFFFFF) via the shared flash_nrf5x page +// cache; saveIfDirty() is the durability point. A full page reclaims the oldest +// (stranded live entries re-appended, then erased). Flash access holds spiLock — +// the page cache is shared with InternalFS/LittleFS. + +bool WarmNodeStore::ringReadHeader(uint8_t page, WarmPageHeader &h) const +{ + flash_nrf5x_read(&h, WARM_FLASH_PAGE_ADDR(page), sizeof(h)); + return h.magic == WARM_RING_MAGIC && h.seq != 0xFFFFFFFFu; +} + +// Caller holds spiLock. +void WarmNodeStore::ringOpenPage(uint8_t page) +{ + // Drop any cached state for the page before the real erase, so a later + // cache flush can't resurrect stale bytes. + flash_nrf5x_flush(); + flash_nrf5x_erase(WARM_FLASH_PAGE_ADDR(page)); + WarmPageHeader h; + h.magic = WARM_RING_MAGIC; + h.seq = nextSeq++; + flash_nrf5x_write(WARM_FLASH_PAGE_ADDR(page), &h, sizeof(h)); + activePage = page; + writeSlot = 0; +} + +// Caller holds spiLock. May recurse once via ringAppend if the stranded set +// fills the fresh page exactly — bounded by WARM_NODE_COUNT <= 2*kRecordsPerPage. +void WarmNodeStore::ringRotate() +{ + uint8_t target = 0; + if (activePage != kNoPage) { + // Lowest-seq valid page, preferring erased pages; never the active one + uint32_t bestSeq = 0; + bool found = false; + for (uint8_t p = 0; p < WARM_FLASH_PAGES; p++) { + if (p == activePage) + continue; + WarmPageHeader h; + if (!ringReadHeader(p, h)) { + target = p; // erased/invalid page: free real estate, take it + found = true; + break; + } + if (!found || static_cast(h.seq - bestSeq) < 0) { + target = p; + bestSeq = h.seq; + found = true; + } + } + } + + // Capture live entries stranded in the page we're about to erase + int stranded[WARM_NODE_COUNT] = {}; + int nStranded = 0; + for (size_t i = 0; i < WARM_NODE_COUNT; i++) { + if (entries[i].num && pageOf[i] == target) + stranded[nStranded++] = static_cast(i); + if (pageOf[i] == target) + pageOf[i] = kNoPage; + } + + ringOpenPage(target); + + for (int k = 0; k < nStranded; k++) + ringAppend(entries[stranded[k]], stranded[k]); +} + +// Caller holds spiLock. +void WarmNodeStore::ringAppend(const WarmNodeEntry &rec, int storeSlot) +{ + if (activePage == kNoPage || writeSlot >= kRecordsPerPage) + ringRotate(); + const uint32_t addr = + WARM_FLASH_PAGE_ADDR(activePage) + sizeof(WarmPageHeader) + static_cast(writeSlot) * sizeof(WarmNodeEntry); + flash_nrf5x_write(addr, &rec, sizeof(rec)); + writeSlot++; + if (storeSlot >= 0) + pageOf[storeSlot] = activePage; + dirty = true; +} + +void WarmNodeStore::persistEntry(const WarmNodeEntry &e) +{ + concurrency::LockGuard g(spiLock); + ringAppend(e, static_cast(&e - entries)); +} + +void WarmNodeStore::persistRemove(NodeNum num, int storeSlot) +{ + if (storeSlot >= 0 && storeSlot < static_cast(WARM_NODE_COUNT)) + pageOf[storeSlot] = 0xFF; + WarmNodeEntry tomb; + memset(&tomb, 0, sizeof(tomb)); + tomb.num = num; + tomb.last_heard = WARM_RING_TOMBSTONE; + concurrency::LockGuard g(spiLock); + ringAppend(tomb, -1); +} + +void WarmNodeStore::persistClear() +{ + concurrency::LockGuard g(spiLock); + flash_nrf5x_flush(); + for (uint8_t p = 0; p < WARM_FLASH_PAGES; p++) + flash_nrf5x_erase(WARM_FLASH_PAGE_ADDR(p)); + activePage = 0xFF; + writeSlot = 0; + nextSeq = 1; + dirty = false; // the erased ring already reflects the empty store +} + +void WarmNodeStore::load() +{ + if (!entries) + return; + concurrency::LockGuard g(spiLock); + + // Order valid pages by ascending seq so replay applies oldest first + uint8_t order[WARM_FLASH_PAGES] = {}; + uint32_t seqs[WARM_FLASH_PAGES] = {}; + uint8_t nValid = 0; + uint8_t nCorrupt = 0; + for (uint8_t p = 0; p < WARM_FLASH_PAGES; p++) { + WarmPageHeader h; + if (!ringReadHeader(p, h)) { + // An erased page reads back all-ones; any other magic is a + // partially-written or bit-rotted header we're dropping, so flag it + // rather than silently treating the loss as a clean empty ring. + if (h.magic != 0xFFFFFFFFu) + nCorrupt++; + continue; + } + uint8_t pos = nValid; + while (pos > 0 && static_cast(h.seq - seqs[pos - 1]) < 0) { + order[pos] = order[pos - 1]; + seqs[pos] = seqs[pos - 1]; + pos--; + } + order[pos] = p; + seqs[pos] = h.seq; + nValid++; + } + + if (nValid == 0) { + activePage = 0xFF; + writeSlot = 0; + nextSeq = 1; + if (nCorrupt) + LOG_WARN("WarmStore: ring unreadable (%u bad page(s)), empty", nCorrupt); + else + LOG_INFO("WarmStore: ring empty, starting fresh"); + return; + } + + uint32_t replayed = 0; + for (uint8_t k = 0; k < nValid; k++) { + const uint8_t p = order[k]; + uint16_t slot = 0; + for (; slot < kRecordsPerPage; slot++) { + WarmNodeEntry rec; + flash_nrf5x_read(&rec, WARM_FLASH_PAGE_ADDR(p) + sizeof(WarmPageHeader) + (uint32_t)slot * sizeof(rec), sizeof(rec)); + if (rec.num == 0xFFFFFFFFu) + break; // erased space: end of this page's records (append-only) + if (rec.num == 0) + continue; // unexpected; skip defensively + replayed++; + if (rec.last_heard == WARM_RING_TOMBSTONE) { + WarmNodeEntry *e = find(rec.num); + if (e) { + pageOf[e - entries] = 0xFF; + memset(e, 0, sizeof(*e)); + } + } else { + const WarmNodeEntry *e = place(rec.num, rec.last_heard, rec.public_key); + if (e) + pageOf[e - entries] = p; + } + } + if (k == nValid - 1) { // newest page becomes the active head + activePage = p; + writeSlot = slot; + nextSeq = seqs[k] + 1; + } + } + if (nCorrupt) + LOG_WARN("WarmStore: dropped %u corrupt ring page(s), some nodes lost", nCorrupt); + LOG_INFO("WarmStore: replayed %u ring records -> %u live nodes (page %u, slot %u)", (unsigned)replayed, (unsigned)count(), + activePage, writeSlot); +} + +bool WarmNodeStore::save() +{ + if (!powerHAL_isPowerLevelSafe()) { + LOG_ERROR("Error: trying to save WarmStore on unsafe device power level."); + return false; + } + concurrency::LockGuard g(spiLock); + flash_nrf5x_flush(); + return true; +} + +#else // !NRF52840_XXAA -------------------- + +void WarmNodeStore::persistEntry(const WarmNodeEntry &e) +{ + (void)e; + dirty = true; +} + +void WarmNodeStore::persistRemove(NodeNum num, int storeSlot) +{ + (void)num; + (void)storeSlot; + dirty = true; +} + +void WarmNodeStore::persistClear() +{ + dirty = true; +} + +#ifdef FSCom + +// ---- File persistence: /prefs/warm.dat snapshots ---------------------------- + +// Compact occupied slots to the front of `dst`; returns the count. +static uint16_t packEntries(const WarmNodeEntry *src, WarmNodeEntry *dst) +{ + uint16_t n = 0; + for (size_t i = 0; i < WARM_NODE_COUNT; i++) + if (src[i].num) + dst[n++] = src[i]; + return n; +} + +void WarmNodeStore::load() +{ + if (!entries) + return; + // Clear first — all failure paths below then correctly represent "empty", + // even if load() is called on an already-used instance. + memset(entries, 0, WARM_NODE_COUNT * sizeof(WarmNodeEntry)); + concurrency::LockGuard g(spiLock); + auto f = FSCom.open(warmFileName, FILE_O_READ); + if (!f) + return; + WarmStoreHeader h; + if ((size_t)f.read((uint8_t *)&h, sizeof(h)) != sizeof(h)) { + f.close(); + LOG_WARN("WarmStore: %s header read failed, starting empty", warmFileName); + return; + } + if (h.magic != WARM_STORE_MAGIC || h.entrySize != sizeof(WarmNodeEntry) || h.count > WARM_NODE_COUNT) { + f.close(); + LOG_WARN("WarmStore: %s header invalid (magic=0x%08x entrySize=%u count=%u), starting empty", warmFileName, h.magic, + h.entrySize, h.count); + return; + } + if (h.count) { + const size_t len = (size_t)h.count * sizeof(WarmNodeEntry); + const bool readOk = (size_t)f.read((uint8_t *)entries, len) == len; + f.close(); + if (!readOk) { + LOG_WARN("WarmStore: %s entries read failed, starting empty", warmFileName); + return; + } + if (crc32Buffer(entries, len) != h.crc) { + LOG_WARN("WarmStore: %s CRC mismatch, starting empty", warmFileName); + return; + } + } else { + f.close(); + } + LOG_INFO("WarmStore: loaded %u warm nodes from %s", h.count, warmFileName); +} + +bool WarmNodeStore::save() +{ + if (!entries) + return false; + if (!powerHAL_isPowerLevelSafe()) { + LOG_ERROR("Error: trying to save WarmStore on unsafe device power level."); + return false; + } + + std::vector packed(WARM_NODE_COUNT); + WarmStoreHeader h; + h.magic = WARM_STORE_MAGIC; + h.reserved = 0; + h.count = packEntries(entries, packed.data()); + h.entrySize = sizeof(WarmNodeEntry); + h.crc = crc32Buffer(packed.data(), h.count * sizeof(WarmNodeEntry)); + + concurrency::LockGuard g(spiLock); + FSCom.mkdir("/prefs"); + + auto f = SafeFile(warmFileName, false); + f.write((const uint8_t *)&h, sizeof(h)); + f.write((const uint8_t *)packed.data(), h.count * sizeof(WarmNodeEntry)); + bool ok = f.close(); + if (!ok) + LOG_ERROR("WarmStore: can't write %s", warmFileName); + else + LOG_DEBUG("WarmStore: saved %u warm nodes to %s", h.count, warmFileName); + return ok; +} + +#else + +void WarmNodeStore::load() {} +bool WarmNodeStore::save() +{ + return true; +} + +#endif // FSCom +#endif // NRF52840_XXAA + +#endif // WARM_NODE_COUNT > 0 diff --git a/src/mesh/WarmNodeStore.h b/src/mesh/WarmNodeStore.h new file mode 100644 index 000000000..c5320317b --- /dev/null +++ b/src/mesh/WarmNodeStore.h @@ -0,0 +1,130 @@ +#pragma once + +#include "MeshTypes.h" +#include "mesh-pb-constants.h" +#include +#include + +// Verbose tracing for the warm-store migration + NodeDB self-care. Per-event / +// per-boot chatter routes through this so it can be silenced in one place (set +// to 0) once they're proven; genuine LOG_WARN anomalies stay unconditional. +#ifndef MESHTASTIC_NODEDB_MIGRATION_VERBOSE +#define MESHTASTIC_NODEDB_MIGRATION_VERBOSE 0 +#endif +#if MESHTASTIC_NODEDB_MIGRATION_VERBOSE +#define LOG_MIGRATION(...) LOG_INFO(__VA_ARGS__) +#else +#define LOG_MIGRATION(...) ((void)0) +#endif + +#if WARM_NODE_COUNT > 0 + +/** + * Warm ("long-tail") node tier. + * + * Minimal identity record (NodeNum, last_heard, Curve25519 public key) for nodes + * evicted from the hot NodeInfoLite store, so DMs to/from them keep encrypting — + * the key is expensive to re-learn, the rest rebuilds from traffic in seconds. + * Flat fixed array, linear scan (only on hot-store misses), LRU by last_heard + * with keyed entries outranking keyless. + * + * Persistence: nRF52840 uses a 12 KB raw-flash record-ring below LittleFS + * (append + replay + compact-on-rotate — see the backend in WarmNodeStore.cpp, + * link-guarded by nrf52840_s140_v7.ld). Everywhere else: /prefs/warm.dat. + */ +struct WarmNodeEntry { + NodeNum num; // 0 = empty slot + uint32_t last_heard; // recency for LRU ordering + uint8_t public_key[32]; // all-zero = no key (a real key is never all-zero) +}; +static_assert(sizeof(WarmNodeEntry) == 40, "WarmNodeEntry must stay 40 B — persistence format depends on it"); + +// Gated on NRF52840_XXAA: the ring sits at 0xEA000 +// valid only on the 1 MB-flash nRF52840. +#if defined(NRF52840_XXAA) +#define WARM_FLASH_PAGE_SIZE 4096u +#define WARM_FLASH_PAGES 3u +#define WARM_FLASH_REGION_BASE (0xED000u - WARM_FLASH_PAGES * WARM_FLASH_PAGE_SIZE) // 0xEA000 +#define WARM_FLASH_PAGE_ADDR(i) (WARM_FLASH_REGION_BASE + (i)*WARM_FLASH_PAGE_SIZE) +#endif + +class WarmNodeStore +{ + public: + WarmNodeStore(); + ~WarmNodeStore(); + WarmNodeStore(const WarmNodeStore &) = delete; + WarmNodeStore &operator=(const WarmNodeStore &) = delete; + + /// Remember an evicted hot node. Keyless candidates never displace keyed + /// entries; otherwise the oldest (keyless-first) entry is replaced. + /// @return true if the node was stored or updated + bool absorb(NodeNum num, uint32_t lastHeard, const uint8_t *key32 /* may be NULL */); + + /// Find and remove an entry (used when the node is re-admitted to the hot store). + bool take(NodeNum num, WarmNodeEntry &out); + + /// Copy the 32-byte public key for a node, if we have one. + bool copyKey(NodeNum num, uint8_t out[32]) const; + + bool contains(NodeNum num) const; + void remove(NodeNum num); + void clear(); + size_t count() const; + size_t capacity() const { return entries ? WARM_NODE_COUNT : 0; } + +#if MESHTASTIC_NODEDB_MIGRATION_VERBOSE + /// Debug: dump every live warm entry (num / last_heard / has-key) to the + /// console. Compiled out unless MESHTASTIC_NODEDB_MIGRATION_VERBOSE. + void dumpToLog(const char *reason = "dump") const; +#endif + + /// Load persisted entries (called once at boot, after the node DB loads). + void load(); + /// Durability point, piggybacked on the node-database save cadence. On the + /// ring backend this flushes the shared flash page cache; on the file + /// backend it writes the warm.dat snapshot. + bool saveIfDirty(); + + private: + WarmNodeEntry *entries = nullptr; // WARM_NODE_COUNT slots; PSRAM on ESP32 when available + bool dirty = false; + + WarmNodeEntry *find(NodeNum num) const; + // Internal slot-placement shared by absorb() and ring replay: applies the + // keyed-first admission policy without touching persistence. + WarmNodeEntry *place(NodeNum num, uint32_t lastHeard, const uint8_t *key32); + + // Persistence hooks called from the mutation paths. File backend: mark + // dirty. Ring backend: append an upsert/tombstone record (+ mark dirty). + void persistEntry(const WarmNodeEntry &e); // e must point into entries[] + void persistRemove(NodeNum num, int storeSlot); + void persistClear(); + +#if defined(NRF52840_XXAA) + // nRF52840 raw-flash record-ring state. + struct WarmPageHeader { + uint32_t magic; // WARM_RING_MAGIC + uint32_t seq; // page generation; 0xFFFFFFFF = erased/unused + }; + static_assert(sizeof(WarmPageHeader) == 8, "page header is part of the flash format"); + static constexpr uint16_t kRecordsPerPage = (WARM_FLASH_PAGE_SIZE - sizeof(WarmPageHeader)) / sizeof(WarmNodeEntry); // 102 + static_assert(WARM_NODE_COUNT <= 2 * ((WARM_FLASH_PAGE_SIZE - 8) / 40), "live set must fit the ring with one page reclaimed"); + + static constexpr uint8_t kNoPage = 0xFF; // "no page" sentinel for activePage / pageOf[] + + uint8_t activePage = kNoPage; // no page opened yet (fresh/erased ring) + uint16_t writeSlot = 0; // next free record slot in the active page + uint32_t nextSeq = 1; // seq for the next page opened + uint8_t pageOf[WARM_NODE_COUNT]; // flash page holding each RAM slot's newest record; kNoPage = none + + void ringAppend(const WarmNodeEntry &rec, int storeSlot /* -1 for tombstones */); + void ringRotate(); // reclaim oldest page, compacting stranded live entries + void ringOpenPage(uint8_t page); // erase + write header (seq = nextSeq++) + bool ringReadHeader(uint8_t page, WarmPageHeader &h) const; +#endif + + bool save(); +}; + +#endif // WARM_NODE_COUNT > 0 diff --git a/src/mesh/mesh-pb-constants.h b/src/mesh/mesh-pb-constants.h index b4d999011..c7edb98a3 100644 --- a/src/mesh/mesh-pb-constants.h +++ b/src/mesh/mesh-pb-constants.h @@ -48,39 +48,40 @@ static_assert(sizeof(meshtastic_NodeInfoLite) <= 130, "NodeInfoLite size increas #define MESHTASTIC_EXCLUDE_POSITIONDB 1 #else #define MESHTASTIC_EXCLUDE_POSITIONDB 0 -#endif -#endif +#endif // STM32WL +#endif // MESHTASTIC_EXCLUDE_POSITIONDB #ifndef MESHTASTIC_EXCLUDE_TELEMETRYDB #if defined(ARCH_STM32WL) #define MESHTASTIC_EXCLUDE_TELEMETRYDB 1 #else #define MESHTASTIC_EXCLUDE_TELEMETRYDB 0 -#endif -#endif +#endif // STM32WL +#endif // MESHTASTIC_EXCLUDE_TELEMETRYDB #ifndef MESHTASTIC_EXCLUDE_ENVIRONMENTDB #if defined(ARCH_STM32WL) #define MESHTASTIC_EXCLUDE_ENVIRONMENTDB 1 #else #define MESHTASTIC_EXCLUDE_ENVIRONMENTDB 0 -#endif -#endif +#endif // STM32WL +#endif // MESHTASTIC_EXCLUDE_ENVIRONMENTDB #ifndef MESHTASTIC_EXCLUDE_STATUSDB #if defined(ARCH_STM32WL) || defined(MESHTASTIC_EXCLUDE_STATUS) #define MESHTASTIC_EXCLUDE_STATUSDB 1 #else #define MESHTASTIC_EXCLUDE_STATUSDB 0 -#endif -#endif +#endif // STM32WL +#endif // MESHTASTIC_EXCLUDE_STATUSDB -/// max number of nodes allowed in the nodeDB +/// Max nodes in the hot store (full NodeInfoLite). Evicted nodes' identities +/// live in the warm tier (WARM_NODE_COUNT). nRF52840 caps at 120 to keep +/// nodes.proto inside the stock 28 KB LittleFS; flash-rich platforms (ESP32-S3, +/// portduino) keep their larger hot store and lean on warm only for the tail. #ifndef MAX_NUM_NODES #if defined(ARCH_STM32WL) #define MAX_NUM_NODES 10 -#elif defined(ARCH_NRF52) -#define MAX_NUM_NODES 150 #elif defined(CONFIG_IDF_TARGET_ESP32S3) #include "Esp.h" static inline int get_max_num_nodes() @@ -95,10 +96,43 @@ static inline int get_max_num_nodes() } } #define MAX_NUM_NODES get_max_num_nodes() +#elif defined(ARCH_PORTDUINO) +#define MAX_NUM_NODES 250 // native host: no flash/RAM constraint; match the ESP32-S3 top tier #else -#define MAX_NUM_NODES 100 -#endif -#endif +#define MAX_NUM_NODES 120 // nRF52840 (28 KB LittleFS) and generic ESP32 +#endif // platform +#endif // MAX_NUM_NODES + +/// Per-map cap (position/telemetry/environment/status): only the freshest +/// MAX_SATELLITE_NODES nodes keep satellite payloads, the rest just the +/// NodeInfoLite header. RAM-bound (the maps are internal-SRAM, not PSRAM), so +/// flash-rich hosts get a cap >= their hot store (satellites for every node, as +/// before the cap existed) while constrained parts stay at 40. +#ifndef MAX_SATELLITE_NODES +#if defined(CONFIG_IDF_TARGET_ESP32S3) || defined(ARCH_PORTDUINO) +#define MAX_SATELLITE_NODES 250 +#else +#define MAX_SATELLITE_NODES 40 // nRF52840 (28 KB LittleFS) and generic ESP32 +#endif // platform +#endif // MAX_SATELLITE_NODES + +/// Warm tier: 40 B {num, last_heard, public_key} records kept for evicted nodes +/// so DMs to/from them keep decrypting. 0 disables it; size is per-platform +/// below, persisted to /prefs/warm.dat (or the nRF52840 raw-flash ring). +#ifndef WARM_NODE_COUNT +#if defined(ARCH_STM32WL) +#define WARM_NODE_COUNT 0 +#elif defined(NRF52840_XXAA) +// Keyed on the NRF52840_XXAA build flag, not ARCH_NRF52: the latter (from +// architecture.h via configuration.h) isn't defined this early in every include +// chain. Backed by the raw-flash ring below LittleFS — see WarmNodeStore.h. +#define WARM_NODE_COUNT 200 +#elif defined(CONFIG_IDF_TARGET_ESP32S3) +#define WARM_NODE_COUNT 2000 // PSRAM-backed when available; warm.dat ~80 KB +#else +#define WARM_NODE_COUNT 320 +#endif // platform +#endif // WARM_NODE_COUNT /// Max number of channels allowed #define MAX_NUM_CHANNELS (member_size(meshtastic_ChannelFile, channels) / member_size(meshtastic_ChannelFile, channels[0])) @@ -125,8 +159,8 @@ static inline int get_max_num_nodes() #define TRAFFIC_MANAGEMENT_CACHE_SIZE 1000 #else #define TRAFFIC_MANAGEMENT_CACHE_SIZE 0 -#endif -#endif +#endif // HAS_TRAFFIC_MANAGEMENT +#endif // TRAFFIC_MANAGEMENT_CACHE_SIZE /// helper function for encoding a record as a protobuf, any failures to encode are fatal and we will panic /// returns the encoded packet size diff --git a/src/modules/AdminModule.cpp b/src/modules/AdminModule.cpp index 8715e202d..af4e3f969 100644 --- a/src/modules/AdminModule.cpp +++ b/src/modules/AdminModule.cpp @@ -181,8 +181,11 @@ bool AdminModule::handleReceivedProtobuf(const meshtastic_MeshPacket &mp, meshta // without the user doing so deliberately. LOG_INFO("PKC admin valid, but not auto-favoriting node %x because role==CLIENT_BASE", mp.from); } else { - LOG_INFO("PKC admin valid. Auto-favoriting node %x", mp.from); - nodeInfoLiteSetBit(remoteNode, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true); + if (nodeDB->setProtectedFlag(remoteNode, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true)) { + LOG_INFO("PKC admin valid. Auto-favoriting node %x", mp.from); + } else { + LOG_WARN("PKC admin valid, but auto-favorite refused for node %x (protected-node cap)", mp.from); + } } } } else { @@ -472,10 +475,13 @@ bool AdminModule::handleReceivedProtobuf(const meshtastic_MeshPacket &mp, meshta LOG_INFO("Client received set_favorite_node command"); meshtastic_NodeInfoLite *node = nodeDB->getMeshNode(r->set_favorite_node); if (node != NULL) { - nodeInfoLiteSetBit(node, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true); - saveChanges(SEGMENT_NODEDATABASE, false); - if (screen) - screen->setFrames(graphics::Screen::FOCUS_PRESERVE); // <-- Rebuild screens + if (nodeDB->setProtectedFlag(node, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true)) { + saveChanges(SEGMENT_NODEDATABASE, false); + if (screen) + screen->setFrames(graphics::Screen::FOCUS_PRESERVE); // <-- Rebuild screens + } else if (mp.from == 0) { // local request from the phone — tell the user why it didn't take + sendWarning(NodeDB::PROTECTED_CAP_WARN_FMT, "favorite", r->set_favorite_node, MAX_NUM_NODES - 2); + } } break; } @@ -492,13 +498,17 @@ bool AdminModule::handleReceivedProtobuf(const meshtastic_MeshPacket &mp, meshta } case meshtastic_AdminMessage_set_ignored_node_tag: { LOG_INFO("Client received set_ignored_node command"); - meshtastic_NodeInfoLite *node = nodeDB->getMeshNode(r->set_ignored_node); + // Unlike the sibling node-targeted admin commands, create the entry if + // it's absent so the block sticks for a node we've not heard from yet + // (e.g. one a remote admin asks us to block) with no NodeInfo or key. + meshtastic_NodeInfoLite *node = nodeDB->getOrCreateMeshNode(r->set_ignored_node); if (node != NULL) { - nodeInfoLiteSetBit(node, NODEINFO_BITFIELD_IS_IGNORED_MASK, true); - nodeDB->eraseNodeSatellites(node->num); - node->public_key.size = 0; - memset(node->public_key.bytes, 0, sizeof(node->public_key.bytes)); - saveChanges(SEGMENT_NODEDATABASE, false); + if (nodeDB->setProtectedFlag(node, NODEINFO_BITFIELD_IS_IGNORED_MASK, true)) { + nodeDB->eraseNodeSatellites(node->num); + saveChanges(SEGMENT_NODEDATABASE, false); + } else if (mp.from == 0) { // local request from the phone — tell the user why it didn't take + sendWarning(NodeDB::PROTECTED_CAP_WARN_FMT, "ignore", r->set_ignored_node, MAX_NUM_NODES - 2); + } } break; } @@ -1405,6 +1415,13 @@ void AdminModule::handleGetNodeRemoteHardwarePins(const meshtastic_MeshPacket &r void AdminModule::handleGetDeviceMetadata(const meshtastic_MeshPacket &req) { +#if WARM_NODE_COUNT > 0 && MESHTASTIC_NODEDB_MIGRATION_VERBOSE + // Debug aid: dump the warm tier to the console on a local metadata request + // (e.g. `meshtastic --info` over USB/BLE). Gated to req.from == 0 so remote + // or admin polling can't spam the console. + if (nodeDB && req.from == 0) + nodeDB->warmStore.dumpToLog("admin get_metadata"); +#endif meshtastic_AdminMessage r = meshtastic_AdminMessage_init_default; r.get_device_metadata_response = getDeviceMetadata(); r.which_payload_variant = meshtastic_AdminMessage_get_device_metadata_response_tag; diff --git a/src/platform/nrf52/nrf52840_s140_v6.ld b/src/platform/nrf52/nrf52840_s140_v6.ld new file mode 100644 index 000000000..c8dac4e55 --- /dev/null +++ b/src/platform/nrf52/nrf52840_s140_v6.ld @@ -0,0 +1,46 @@ +/* Linker script to configure memory regions. */ + +SEARCH_DIR(.) +GROUP(-lgcc -lc -lnosys) + +MEMORY +{ + /* App region ends at 0xEA000, not 0xED000: the 12 KB warm-node-store + * record-ring (3 x 4 KB pages, src/mesh/WarmNodeStore.h) occupies + * 0xEA000-0xED000, directly below the stock LittleFS partition. Boards on + * the framework-default linker script are covered by the post-link guard + * in extra_scripts/nrf52_warm_region.py instead. + * + * S140 v6.x app region starts at 0x26000 (152 KB SoftDevice); v7.x uses + * 0x27000. All other boundaries are identical — see nrf52840_s140_v7.ld. */ + FLASH (rx) : ORIGIN = 0x26000, LENGTH = 0xEA000 - 0x26000 + + /* SRAM required by Softdevice depend on + * - Attribute Table Size (Number of Services and Characteristics) + * - Vendor UUID count + * - Max ATT MTU + * - Concurrent connection peripheral + central + secure links + * - Event Len, HVN queue, Write CMD queue + */ + RAM (rwx) : ORIGIN = 0x20006000, LENGTH = 0x20040000 - 0x20006000 +} + +SECTIONS +{ + . = ALIGN(4); + .svc_data : + { + PROVIDE(__start_svc_data = .); + KEEP(*(.svc_data)) + PROVIDE(__stop_svc_data = .); + } > RAM + + .fs_data : + { + PROVIDE(__start_fs_data = .); + KEEP(*(.fs_data)) + PROVIDE(__stop_fs_data = .); + } > RAM +} INSERT AFTER .data; + +INCLUDE "nrf52_common.ld" diff --git a/src/platform/nrf52/nrf52840_s140_v7.ld b/src/platform/nrf52/nrf52840_s140_v7.ld index 6aaeb4034..4546b4a7a 100644 --- a/src/platform/nrf52/nrf52840_s140_v7.ld +++ b/src/platform/nrf52/nrf52840_s140_v7.ld @@ -5,7 +5,12 @@ GROUP(-lgcc -lc -lnosys) MEMORY { - FLASH (rx) : ORIGIN = 0x27000, LENGTH = 0xED000 - 0x27000 + /* App region ends at 0xEA000, not 0xED000: the 12 KB warm-node-store + * record-ring (3 x 4 KB pages, src/mesh/WarmNodeStore.h) occupies + * 0xEA000-0xED000, directly below the stock LittleFS partition. Boards on + * the framework-default linker script are covered by the post-link guard + * in extra_scripts/nrf52_warm_region.py instead. */ + FLASH (rx) : ORIGIN = 0x27000, LENGTH = 0xEA000 - 0x27000 /* SRAM required by Softdevice depend on * - Attribute Table Size (Number of Services and Characteristics) diff --git a/test/test_nodedb_blocked/test_main.cpp b/test/test_nodedb_blocked/test_main.cpp new file mode 100644 index 000000000..2bbe9fb6b --- /dev/null +++ b/test/test_nodedb_blocked/test_main.cpp @@ -0,0 +1,193 @@ +// Tests for the NodeDB hot-store migration and favourite/ignored (blocked) +// retention paths — src/mesh/NodeDB.cpp. +#include "MeshTypes.h" // BEFORE TestUtil.h — provides WARM_NODE_COUNT / MAX_NUM_NODES via mesh-pb-constants.h +#include "TestUtil.h" +#include + +#if defined(ARCH_PORTDUINO) +#define NDB_TEST_ENTRY extern "C" +#else +#define NDB_TEST_ENTRY +#endif + +// The migration demotes overflow into the warm tier, so these tests need it. +#if WARM_NODE_COUNT > 0 + +#include "mesh/NodeDB.h" +#include + +// Subclass shim: exposes the private maintenance paths (via the friend +// declaration in NodeDB.h) and lets a test own the hot store directly +// (meshNodes/numMeshNodes are public). Declared at global scope so it matches +// `friend class NodeDBTestShim` — an anonymous-namespace class would not. +class NodeDBTestShim : public NodeDB +{ + public: + void runDemote() { demoteOldestHotNodesToWarm(); } + void runCleanup() { cleanupMeshDB(); } + + void clearHot() + { + meshNodes->clear(); + numMeshNodes = 0; + } + + void push(NodeNum num, uint32_t lastHeard, bool favorite, bool ignored, bool withUser, bool withKey) + { + meshtastic_NodeInfoLite n = meshtastic_NodeInfoLite_init_zero; + n.num = num; + n.last_heard = lastHeard; + if (favorite) + nodeInfoLiteSetBit(&n, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true); + if (ignored) + nodeInfoLiteSetBit(&n, NODEINFO_BITFIELD_IS_IGNORED_MASK, true); + if (withUser) + nodeInfoLiteSetBit(&n, NODEINFO_BITFIELD_HAS_USER_MASK, true); + if (withKey) { + n.public_key.size = 32; + memset(n.public_key.bytes, static_cast(num & 0xff), 32); + n.public_key.bytes[0] = 0x01; // ensure non-zero (all-zero == "no key") + } + meshNodes->push_back(n); + numMeshNodes = meshNodes->size(); + } + + // Index 0 is our own node; the eviction/migration scans treat it as self. + void seedSelf() { push(0x0BADF00D, 0xFFFFFFFFu, false, false, /*withUser=*/true, /*withKey=*/false); } +}; + +namespace +{ + +NodeDBTestShim *db = nullptr; + +bool warmHasKey(NodeNum n) +{ + meshtastic_NodeInfoLite_public_key_t k = {0, {0}}; + return db->copyPublicKey(n, k) && k.size == 32; +} + +} // namespace + +void setUp(void) +{ + db->clearHot(); +} +void tearDown(void) {} + +// Migration: a database from a larger-cap build trims to MAX_NUM_NODES; the +// oldest non-protected nodes are demoted into the warm tier (keys preserved), +// while self, favourites and ignored survive even when they are the oldest. +static void test_migration_demotesOldestKeepsKeepersAndSelf(void) +{ + db->seedSelf(); + const int extra = MAX_NUM_NODES + 30; // overflow well past the MAX-2 cap + for (int i = 1; i <= extra; i++) { + const bool fav = (i == 1); // oldest, but a favourite + const bool ign = (i == 2); // 2nd-oldest, but blocked + db->push(2000 + i, /*last_heard=*/i, fav, ign, /*withUser=*/true, /*withKey=*/true); + } + + db->runDemote(); + + TEST_ASSERT_EQUAL_INT(MAX_NUM_NODES, (int)db->getNumMeshNodes()); + TEST_ASSERT_NOT_NULL(db->getMeshNode(0x0BADF00D)); // self retained + TEST_ASSERT_NOT_NULL(db->getMeshNode(2000 + 1)); // oldest favourite retained + TEST_ASSERT_NOT_NULL(db->getMeshNode(2000 + 2)); // oldest ignored retained + TEST_ASSERT_NOT_NULL(db->getMeshNode(2000 + extra)); // freshest retained + TEST_ASSERT_NULL(db->getMeshNode(2000 + 3)); // oldest non-protected demoted out of hot + TEST_ASSERT_TRUE(warmHasKey(2000 + 3)); // ...but its key kept in the warm tier +} + +// Favourite handling: a favourite is never the eviction victim, even when it is +// the oldest node in a full hot store. +static void test_eviction_preservesFavorite(void) +{ + db->seedSelf(); + for (int i = 1; i < MAX_NUM_NODES; i++) { // fill to MAX_NUM_NODES total (incl. self) + const bool fav = (i == 1); // oldest non-self, favourite + db->push(3000 + i, /*last_heard=*/i, fav, false, /*withUser=*/true, /*withKey=*/true); + } + TEST_ASSERT_EQUAL_INT(MAX_NUM_NODES, (int)db->getNumMeshNodes()); // full + + TEST_ASSERT_NOT_NULL(db->getOrCreateMeshNode(0x99990000)); // forces an eviction + + TEST_ASSERT_NOT_NULL(db->getMeshNode(3000 + 1)); // favourite survived despite being oldest + TEST_ASSERT_NULL(db->getMeshNode(3000 + 2)); // oldest non-favourite evicted + TEST_ASSERT_NOT_NULL(db->getMeshNode(0x99990000)); +} + +// Ignored handling: an ignored node survives eviction (like a favourite), and is +// never purged by cleanupMeshDB even with no user info (a block set by bare ID). +static void test_ignored_survivesEvictionAndCleanup(void) +{ + // (a) eviction protection + db->clearHot(); + db->seedSelf(); + for (int i = 1; i < MAX_NUM_NODES; i++) { + const bool ign = (i == 1); // oldest non-self, blocked + db->push(4000 + i, /*last_heard=*/i, false, ign, /*withUser=*/true, /*withKey=*/true); + } + TEST_ASSERT_NOT_NULL(db->getOrCreateMeshNode(0x88880000)); + TEST_ASSERT_NOT_NULL(db->getMeshNode(4000 + 1)); // blocked node survived + TEST_ASSERT_NULL(db->getMeshNode(4000 + 2)); // oldest non-blocked evicted + + // (b) cleanup protection — ignored kept without user info, plain no-user purged + db->clearHot(); + db->seedSelf(); + db->push(5000, 100, false, /*ignored=*/true, /*withUser=*/false, false); + db->push(5001, 100, false, false, /*withUser=*/false, false); + db->runCleanup(); + TEST_ASSERT_NOT_NULL(db->getMeshNode(5000)); // blocked-by-ID kept despite no user info + TEST_ASSERT_NULL(db->getMeshNode(5001)); // ordinary no-user node purged +} + +// Protected-node cap: at most MAX_NUM_NODES-2 nodes may be protected, so >=2 +// evictable slots always remain. setProtectedFlag refuses once the cap is hit. +static void test_protectedCap_refusesBeyondLimit(void) +{ + db->seedSelf(); + for (int i = 0; i < MAX_NUM_NODES - 2; i++) + db->push(6000 + i, 100, /*favorite=*/true, false, /*withUser=*/true, false); + TEST_ASSERT_EQUAL_INT(MAX_NUM_NODES - 2, db->numProtectedNodes()); + + db->push(7000, 100, false, false, /*withUser=*/true, false); + meshtastic_NodeInfoLite *fresh = db->getMeshNode(7000); + TEST_ASSERT_NOT_NULL(fresh); + TEST_ASSERT_FALSE(db->setProtectedFlag(fresh, NODEINFO_BITFIELD_IS_IGNORED_MASK, true)); // refused at cap + TEST_ASSERT_FALSE(nodeInfoLiteIsIgnored(fresh)); // unchanged + TEST_ASSERT_EQUAL_INT(MAX_NUM_NODES - 2, db->numProtectedNodes()); + + // Adding another flag to an already-protected node doesn't grow the set, so + // it's still allowed at the cap. + meshtastic_NodeInfoLite *already = db->getMeshNode(6000); + TEST_ASSERT_TRUE(db->setProtectedFlag(already, NODEINFO_BITFIELD_IS_IGNORED_MASK, true)); +} + +NDB_TEST_ENTRY void setup() +{ + initializeTestEnvironment(); + db = new NodeDBTestShim(); + nodeDB = db; + + UNITY_BEGIN(); + RUN_TEST(test_migration_demotesOldestKeepsKeepersAndSelf); + RUN_TEST(test_eviction_preservesFavorite); + RUN_TEST(test_ignored_survivesEvictionAndCleanup); + RUN_TEST(test_protectedCap_refusesBeyondLimit); + exit(UNITY_END()); +} +NDB_TEST_ENTRY void loop() {} + +#else // WARM_NODE_COUNT == 0 — nothing to exercise here + +void setUp(void) {} +void tearDown(void) {} +NDB_TEST_ENTRY void setup() +{ + UNITY_BEGIN(); + exit(UNITY_END()); +} +NDB_TEST_ENTRY void loop() {} + +#endif diff --git a/test/test_warm_store/test_main.cpp b/test/test_warm_store/test_main.cpp new file mode 100644 index 000000000..3c8455883 --- /dev/null +++ b/test/test_warm_store/test_main.cpp @@ -0,0 +1,211 @@ +// Unit tests for the warm ("long-tail") node tier — src/mesh/WarmNodeStore.cpp. +// Covers admission/eviction policy (keyed entries outrank keyless), take() +// rehydration semantics, and a tolerant persistence round trip. +#include "MeshTypes.h" // BEFORE TestUtil.h — provides WARM_NODE_COUNT via mesh-pb-constants.h +#include "TestUtil.h" +#include + +#if defined(ARCH_PORTDUINO) +#define WS_TEST_ENTRY extern "C" +#else +#define WS_TEST_ENTRY +#endif + +#if WARM_NODE_COUNT > 0 + +#include "mesh/WarmNodeStore.h" +#include + +namespace +{ + +void makeKey(uint8_t out[32], uint8_t seed) +{ + memset(out, 0, 32); + out[0] = seed; + out[31] = seed ^ 0xA5; +} + +} // namespace + +void setUp(void) {} +void tearDown(void) {} + +void test_ws_absorb_and_copyKey_roundTrip() +{ + WarmNodeStore ws; + uint8_t key[32], got[32]; + makeKey(key, 7); + TEST_ASSERT_TRUE(ws.absorb(0x100, 1000, key)); + TEST_ASSERT_TRUE(ws.contains(0x100)); + TEST_ASSERT_TRUE(ws.copyKey(0x100, got)); + TEST_ASSERT_EQUAL_MEMORY(key, got, 32); + TEST_ASSERT_EQUAL(1, ws.count()); +} + +void test_ws_keylessEntry_hasNoKey() +{ + WarmNodeStore ws; + uint8_t got[32]; + TEST_ASSERT_TRUE(ws.absorb(0x200, 1000, NULL)); + TEST_ASSERT_TRUE(ws.contains(0x200)); + TEST_ASSERT_FALSE(ws.copyKey(0x200, got)); +} + +void test_ws_absorb_rejectsNodeNumZero() +{ + WarmNodeStore ws; + TEST_ASSERT_FALSE(ws.absorb(0, 1000, NULL)); + TEST_ASSERT_EQUAL(0, ws.count()); +} + +void test_ws_absorb_updatesExistingEntry() +{ + WarmNodeStore ws; + uint8_t key[32], got[32]; + makeKey(key, 9); + TEST_ASSERT_TRUE(ws.absorb(0x300, 1000, NULL)); + TEST_ASSERT_TRUE(ws.absorb(0x300, 2000, key)); // later eviction learned a key + TEST_ASSERT_EQUAL(1, ws.count()); + TEST_ASSERT_TRUE(ws.copyKey(0x300, got)); + TEST_ASSERT_EQUAL_MEMORY(key, got, 32); +} + +void test_ws_take_removesEntry() +{ + WarmNodeStore ws; + uint8_t key[32]; + makeKey(key, 3); + ws.absorb(0x400, 1234, key); + + WarmNodeEntry e; + TEST_ASSERT_TRUE(ws.take(0x400, e)); + TEST_ASSERT_EQUAL(0x400, e.num); + TEST_ASSERT_EQUAL(1234, e.last_heard); + TEST_ASSERT_EQUAL_MEMORY(key, e.public_key, 32); + TEST_ASSERT_FALSE(ws.contains(0x400)); + TEST_ASSERT_FALSE(ws.take(0x400, e)); + TEST_ASSERT_EQUAL(0, ws.count()); +} + +void test_ws_keylessCandidate_neverEvictsKeyedEntries() +{ + WarmNodeStore ws; + uint8_t key[32]; + // Fill the store entirely with keyed entries + for (size_t i = 0; i < ws.capacity(); i++) { + makeKey(key, (uint8_t)i); + TEST_ASSERT_TRUE(ws.absorb(0x1000 + i, 100 + i, key)); + } + TEST_ASSERT_EQUAL(ws.capacity(), ws.count()); + // A keyless candidate (even a fresh one) must be rejected + TEST_ASSERT_FALSE(ws.absorb(0x9999, 999999, NULL)); + TEST_ASSERT_FALSE(ws.contains(0x9999)); +} + +void test_ws_keyedCandidate_evictsOldestKeylessFirst() +{ + WarmNodeStore ws; + uint8_t key[32]; + makeKey(key, 0x42); + // Fill with keyed entries except two keyless ones in the middle + for (size_t i = 0; i < ws.capacity(); i++) { + const bool keyless = (i == 5 || i == 10); + TEST_ASSERT_TRUE(ws.absorb(0x1000 + i, keyless ? (i == 10 ? 50 : 60) : 10, keyless ? NULL : key)); + } + // Keyed candidate must displace the OLDEST KEYLESS entry (0x100A, ts=50), + // even though every keyed entry is older (ts=10) + uint8_t k2[32]; + makeKey(k2, 0x43); + TEST_ASSERT_TRUE(ws.absorb(0x8888, 70, k2)); + TEST_ASSERT_FALSE(ws.contains(0x1000 + 10)); + TEST_ASSERT_TRUE(ws.contains(0x1000 + 5)); + TEST_ASSERT_TRUE(ws.contains(0x8888)); +} + +void test_ws_keyedCandidate_evictsOldestKeyedWhenNoKeyless() +{ + WarmNodeStore ws; + uint8_t key[32]; + for (size_t i = 0; i < ws.capacity(); i++) { + makeKey(key, (uint8_t)i); + TEST_ASSERT_TRUE(ws.absorb(0x1000 + i, 1000 + i, key)); // 0x1000 is the oldest + } + uint8_t k2[32]; + makeKey(k2, 0x44); + TEST_ASSERT_TRUE(ws.absorb(0x7777, 999999, k2)); + TEST_ASSERT_TRUE(ws.contains(0x7777)); + TEST_ASSERT_FALSE(ws.contains(0x1000)); // oldest keyed evicted + TEST_ASSERT_EQUAL(ws.capacity(), ws.count()); +} + +void test_ws_remove_and_clear() +{ + WarmNodeStore ws; + ws.absorb(0x500, 1, NULL); + ws.absorb(0x501, 2, NULL); + ws.remove(0x500); + TEST_ASSERT_FALSE(ws.contains(0x500)); + TEST_ASSERT_EQUAL(1, ws.count()); + ws.clear(); + TEST_ASSERT_EQUAL(0, ws.count()); +} + +void test_ws_persistence_roundTrip() +{ + WarmNodeStore a; + uint8_t key[32], got[32]; + makeKey(key, 0x55); + a.absorb(0x600, 4242, key); + a.absorb(0x601, 4243, NULL); + if (!a.saveIfDirty()) { + TEST_IGNORE_MESSAGE("Filesystem not available in this test environment"); + return; + } + + WarmNodeStore b; + b.load(); + TEST_ASSERT_TRUE(b.contains(0x600)); + TEST_ASSERT_TRUE(b.contains(0x601)); + TEST_ASSERT_TRUE(b.copyKey(0x600, got)); + TEST_ASSERT_EQUAL_MEMORY(key, got, 32); + + // Cleanup so reruns start fresh + b.clear(); + b.saveIfDirty(); +} + +WS_TEST_ENTRY void setup() +{ + initializeTestEnvironment(); + UNITY_BEGIN(); + RUN_TEST(test_ws_absorb_and_copyKey_roundTrip); + RUN_TEST(test_ws_keylessEntry_hasNoKey); + RUN_TEST(test_ws_absorb_rejectsNodeNumZero); + RUN_TEST(test_ws_absorb_updatesExistingEntry); + RUN_TEST(test_ws_take_removesEntry); + RUN_TEST(test_ws_keylessCandidate_neverEvictsKeyedEntries); + RUN_TEST(test_ws_keyedCandidate_evictsOldestKeylessFirst); + RUN_TEST(test_ws_keyedCandidate_evictsOldestKeyedWhenNoKeyless); + RUN_TEST(test_ws_remove_and_clear); + RUN_TEST(test_ws_persistence_roundTrip); + exit(UNITY_END()); +} + +WS_TEST_ENTRY void loop() {} + +#else + +void setUp(void) {} +void tearDown(void) {} + +WS_TEST_ENTRY void setup() +{ + initializeTestEnvironment(); + UNITY_BEGIN(); + exit(UNITY_END()); +} + +WS_TEST_ENTRY void loop() {} + +#endif diff --git a/variants/nrf52840/nrf52.ini b/variants/nrf52840/nrf52.ini index f7f3d0885..56ab06ef1 100644 --- a/variants/nrf52840/nrf52.ini +++ b/variants/nrf52840/nrf52.ini @@ -15,6 +15,7 @@ extra_scripts = ${env.extra_scripts} extra_scripts/nrf52_extra.py pre:extra_scripts/nrf52_lto.py + extra_scripts/nrf52_warm_region.py ; post-link guard: image must end below the 12 KB warm-store raw-flash region at 0xEA000-0xED000 build_type = release build_flags = diff --git a/variants/nrf52840/nrf52840.ini b/variants/nrf52840/nrf52840.ini index c5590cbc3..610b5b530 100644 --- a/variants/nrf52840/nrf52840.ini +++ b/variants/nrf52840/nrf52840.ini @@ -1,6 +1,10 @@ [nrf52840_base] extends = nrf52_base +; Cap the app image at 0xEA000 (below the WarmNodeStore raw-flash region). +; Boards that have upgraded to S140 v7 override this in their own platformio.ini. +board_build.ldscript = src/platform/nrf52/nrf52840_s140_v6.ld + build_flags = ${nrf52_base.build_flags} -DSERIAL_BUFFER_SIZE=4096 From 68af6b277c5043b5fd50b0542f78e741b0ed8c3e Mon Sep 17 00:00:00 2001 From: Quency-D <55523105+Quency-D@users.noreply.github.com> Date: Thu, 18 Jun 2026 20:07:50 +0800 Subject: [PATCH 13/86] Add Heltec tower v2 board. (#10693) * Add heltec_mesh_tower_v2 board. * Added automatic detection for high and low power versions. * Fix low power consumption issues --- boards/heltec_mesh_tower_v2.json | 54 ++++++ src/configuration.h | 7 + src/mesh/LoRaFEMInterface.cpp | 29 ++++ src/mesh/LoRaFEMInterface.h | 3 +- src/platform/nrf52/architecture.h | 2 + .../heltec_mesh_tower_v2/platformio.ini | 27 +++ .../nrf52840/heltec_mesh_tower_v2/variant.cpp | 61 +++++++ .../nrf52840/heltec_mesh_tower_v2/variant.h | 156 ++++++++++++++++++ 8 files changed, 338 insertions(+), 1 deletion(-) create mode 100644 boards/heltec_mesh_tower_v2.json create mode 100644 variants/nrf52840/heltec_mesh_tower_v2/platformio.ini create mode 100644 variants/nrf52840/heltec_mesh_tower_v2/variant.cpp create mode 100644 variants/nrf52840/heltec_mesh_tower_v2/variant.h diff --git a/boards/heltec_mesh_tower_v2.json b/boards/heltec_mesh_tower_v2.json new file mode 100644 index 000000000..42512262f --- /dev/null +++ b/boards/heltec_mesh_tower_v2.json @@ -0,0 +1,54 @@ +{ + "build": { + "arduino": { + "ldscript": "nrf52840_s140_v6.ld" + }, + "core": "nRF5", + "cpu": "cortex-m4", + "extra_flags": "-DNRF52840_XXAA", + "f_cpu": "64000000L", + "hwids": [ + ["0x239A", "0x4405"], + ["0x239A", "0x0029"], + ["0x239A", "0x002A"], + ["0x239A", "0x0071"] + ], + "usb_product": "HT-n5262", + "mcu": "nrf52840", + "variant": "heltec_mesh_tower_v2", + "variants_dir": "variants", + "bsp": { + "name": "adafruit" + }, + "softdevice": { + "sd_flags": "-DS140", + "sd_name": "s140", + "sd_version": "6.1.1", + "sd_fwid": "0x00B6" + }, + "bootloader": { + "settings_addr": "0xFF000" + } + }, + "connectivity": ["bluetooth"], + "debug": { + "jlink_device": "nRF52840_xxAA", + "onboard_tools": ["jlink"], + "svd_path": "nrf52840.svd", + "openocd_target": "nrf52840-mdk-rs" + }, + "frameworks": ["arduino"], + "name": "Heltec MeshTower V2 (Adafruit BSP)", + "upload": { + "maximum_ram_size": 248832, + "maximum_size": 815104, + "speed": 115200, + "protocol": "nrfutil", + "protocols": ["jlink", "nrfjprog", "nrfutil", "stlink"], + "use_1200bps_touch": true, + "require_upload_port": true, + "wait_for_upload_port": true + }, + "url": "https://heltec.org", + "vendor": "Heltec" +} diff --git a/src/configuration.h b/src/configuration.h index 817204da0..3833f97b3 100644 --- a/src/configuration.h +++ b/src/configuration.h @@ -179,6 +179,13 @@ along with this program. If not, see . #endif #endif +#ifdef USE_KCT8103L_PA_ONLY +#if defined(HELTEC_MESH_TOWER_V2) +#define NUM_PA_POINTS 22 +#define TX_GAIN_LORA 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 10, 10, 10, 10, 10, 10, 10, 10, 10, 9, 8, 7 +#endif +#endif + #ifdef RAK13302 #define NUM_PA_POINTS 22 #define TX_GAIN_LORA 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 8 diff --git a/src/mesh/LoRaFEMInterface.cpp b/src/mesh/LoRaFEMInterface.cpp index d34ed008e..6fe6950ac 100644 --- a/src/mesh/LoRaFEMInterface.cpp +++ b/src/mesh/LoRaFEMInterface.cpp @@ -57,6 +57,11 @@ static void releaseSleepHolds() void LoRaFEMInterface::init(void) { setLnaCanControl(false); // Default is uncontrollable +#if defined(RF_PA_DETECT_PIN) + pinMode(RF_PA_DETECT_PIN, INPUT); + high_power_pa = (digitalRead(RF_PA_DETECT_PIN) == RF_PA_HIGH_POWER_VALUE); + LOG_INFO("Detected %s LoRa PA profile", high_power_pa ? "high-power" : "low-power"); +#endif #ifdef HELTEC_V4 pinMode(LORA_PA_POWER, OUTPUT); digitalWrite(LORA_PA_POWER, HIGH); @@ -119,6 +124,13 @@ void LoRaFEMInterface::init(void) pinMode(LORA_KCT8103L_PA_CTX, OUTPUT); digitalWrite(LORA_KCT8103L_PA_CTX, LOW); // LNA enabled by default setLnaCanControl(true); +#elif defined(USE_KCT8103L_PA_ONLY) + fem_type = KCT8103L_PA; + pinMode(LORA_KCT8103L_EN, OUTPUT); + digitalWrite(LORA_KCT8103L_EN, HIGH); + delay(1); + pinMode(LORA_KCT8103L_TX_RX, OUTPUT); + digitalWrite(LORA_KCT8103L_TX_RX, LOW); #endif } @@ -148,6 +160,9 @@ void LoRaFEMInterface::setSleepModeEnable(void) // shutdown the PA digitalWrite(LORA_KCT8103L_PA_CSD, LOW); digitalWrite(LORA_PA_POWER, LOW); +#elif defined(USE_KCT8103L_PA_ONLY) + // shutdown the PA + digitalWrite(LORA_KCT8103L_EN, LOW); #endif } @@ -173,6 +188,9 @@ void LoRaFEMInterface::setTxModeEnable(void) enableFEMPower(); digitalWrite(LORA_KCT8103L_PA_CSD, HIGH); digitalWrite(LORA_KCT8103L_PA_CTX, HIGH); +#elif defined(USE_KCT8103L_PA_ONLY) + enableFEMPower(); + digitalWrite(LORA_KCT8103L_TX_RX, HIGH); #endif } @@ -206,6 +224,9 @@ void LoRaFEMInterface::setRxModeEnable(void) } else { digitalWrite(LORA_KCT8103L_PA_CTX, HIGH); } +#elif defined(USE_KCT8103L_PA_ONLY) + enableFEMPower(); + digitalWrite(LORA_KCT8103L_TX_RX, LOW); #endif } @@ -247,6 +268,9 @@ void LoRaFEMInterface::setRxModeEnableWhenMCUSleep(void) rtc_gpio_hold_en((gpio_num_t)LORA_KCT8103L_PA_CSD); rtc_gpio_hold_en((gpio_num_t)LORA_KCT8103L_PA_CTX); #endif +#elif defined(USE_KCT8103L_PA_ONLY) + enableFEMPower(); + digitalWrite(LORA_KCT8103L_TX_RX, LOW); #endif } @@ -257,6 +281,11 @@ void LoRaFEMInterface::setLNAEnable(bool enabled) int8_t LoRaFEMInterface::powerConversion(int8_t loraOutputPower) { +#if defined(RF_PA_DETECT_PIN) + if (!high_power_pa) { + return loraOutputPower; + } +#endif #ifdef HELTEC_V4 const uint16_t gc1109_tx_gain[] = {11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 10, 10, 9, 9, 8, 7}; const uint16_t kct8103l_tx_gain[] = {13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 12, 12, 11, 11, 10, 9, 8, 7}; diff --git a/src/mesh/LoRaFEMInterface.h b/src/mesh/LoRaFEMInterface.h index 14220c6e3..f524f51d7 100644 --- a/src/mesh/LoRaFEMInterface.h +++ b/src/mesh/LoRaFEMInterface.h @@ -24,7 +24,8 @@ class LoRaFEMInterface LoRaFEMType fem_type; bool lna_enabled = true; bool lna_can_control = false; + bool high_power_pa = true; }; extern LoRaFEMInterface loraFEMInterface; -#endif \ No newline at end of file +#endif diff --git a/src/platform/nrf52/architecture.h b/src/platform/nrf52/architecture.h index 53ca80969..0c04bbbab 100644 --- a/src/platform/nrf52/architecture.h +++ b/src/platform/nrf52/architecture.h @@ -137,6 +137,8 @@ #define HW_VENDOR meshtastic_HardwareModel_HELTEC_MESH_SOLAR #elif defined(MUZI_BASE) #define HW_VENDOR meshtastic_HardwareModel_MUZI_BASE +#elif defined(HELTEC_MESH_TOWER_V2) +#define HW_VENDOR meshtastic_HardwareModel_HELTEC_MESH_TOWER_V2 #else #define HW_VENDOR meshtastic_HardwareModel_NRF52_UNKNOWN #endif diff --git a/variants/nrf52840/heltec_mesh_tower_v2/platformio.ini b/variants/nrf52840/heltec_mesh_tower_v2/platformio.ini new file mode 100644 index 000000000..05f3f49d4 --- /dev/null +++ b/variants/nrf52840/heltec_mesh_tower_v2/platformio.ini @@ -0,0 +1,27 @@ +; Heltec MeshTower V2 nrf52840/sx1262 device +[env:heltec-mesh-tower-v2] +custom_meshtastic_hw_model = 139 +custom_meshtastic_hw_model_slug = HELTEC_MESH_TOWER_V2 +custom_meshtastic_architecture = nrf52840 +custom_meshtastic_actively_supported = false +custom_meshtastic_support_level = 1 +custom_meshtastic_display_name = Heltec MeshTower V2 +custom_meshtastic_images = heltec-mesh-tower-v2.svg +custom_meshtastic_tags = Heltec + +extends = nrf52840_base +board = heltec_mesh_tower_v2 +board_level = pr +debug_tool = jlink + +# add -DCFG_SYSVIEW if you want to use the Segger systemview tool for OS profiling. +build_flags = ${nrf52840_base.build_flags} + -Ivariants/nrf52840/heltec_mesh_tower_v2 + -DHELTEC_MESH_TOWER_V2 + -D HAS_LORA_FEM=1 + +build_src_filter = ${nrf52_base.build_src_filter} +<../variants/nrf52840/heltec_mesh_tower_v2> +lib_deps = + ${nrf52840_base.lib_deps} + # renovate: datasource=custom.pio depName=ArduinoJson packageName=bblanchon/library/ArduinoJson + bblanchon/ArduinoJson@6.21.6 diff --git a/variants/nrf52840/heltec_mesh_tower_v2/variant.cpp b/variants/nrf52840/heltec_mesh_tower_v2/variant.cpp new file mode 100644 index 000000000..08843c0a6 --- /dev/null +++ b/variants/nrf52840/heltec_mesh_tower_v2/variant.cpp @@ -0,0 +1,61 @@ +/* + Copyright (c) 2014-2015 Arduino LLC. All right reserved. + Copyright (c) 2016 Sandeep Mistry All right reserved. + Copyright (c) 2018, Adafruit Industries (adafruit.com) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "variant.h" +#include "Arduino.h" +#include "nrf.h" +#include "wiring_constants.h" +#include "wiring_digital.h" + +const uint32_t g_ADigitalPinMap[] = { + // P0 - pins 0 and 1 are hardwired for xtal and should never be enabled + 0xff, 0xff, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + + // P1 + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47}; + +void initVariant() +{ + +} + +void variant_shutdown() +{ + nrf_gpio_cfg_default(PIN_GPS_EN); + nrf_gpio_cfg_default(PIN_GPS_PPS); + nrf_gpio_cfg_default(PIN_GPS_RESET); + nrf_gpio_cfg_default(PIN_GPS_STANDBY); + nrf_gpio_cfg_default(GPS_RX_PIN); + nrf_gpio_cfg_default(GPS_TX_PIN); + nrf_gpio_cfg_default(ADC_CTRL); + pinMode(LORA_KCT8103L_EN, OUTPUT); + digitalWrite(LORA_KCT8103L_EN, LOW); + nrf_gpio_cfg_default(LORA_KCT8103L_TX_RX); + nrf_gpio_cfg_default(RF_PA_DETECT_PIN); + nrf_gpio_cfg_default(SX126X_CS); + nrf_gpio_cfg_default(SX126X_DIO1); + nrf_gpio_cfg_default(SX126X_BUSY); + nrf_gpio_cfg_default(SX126X_RESET); + nrf_gpio_cfg_default(PIN_SPI_MISO); + nrf_gpio_cfg_default(PIN_SPI_MOSI); + nrf_gpio_cfg_default(PIN_SPI_SCK); + detachInterrupt(PIN_GPS_PPS); + detachInterrupt(PIN_BUTTON1); +} diff --git a/variants/nrf52840/heltec_mesh_tower_v2/variant.h b/variants/nrf52840/heltec_mesh_tower_v2/variant.h new file mode 100644 index 000000000..1dd0ce63f --- /dev/null +++ b/variants/nrf52840/heltec_mesh_tower_v2/variant.h @@ -0,0 +1,156 @@ +/* + Copyright (c) 2014-2015 Arduino LLC. All right reserved. + Copyright (c) 2016 Sandeep Mistry All right reserved. + Copyright (c) 2018, Adafruit Industries (adafruit.com) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU Lesser General Public License for more details. + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef _VARIANT_HELTEC_NRF_ +#define _VARIANT_HELTEC_NRF_ +/** Master clock frequency */ +#define VARIANT_MCK (64000000ul) + +#define USE_LFXO // Board uses 32khz crystal for LF + +/*---------------------------------------------------------------------------- + * Headers + *----------------------------------------------------------------------------*/ + +#include "WVariant.h" + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +// Number of pins defined in PinDescription array +#define PINS_COUNT (48) +#define NUM_DIGITAL_PINS (48) +#define NUM_ANALOG_INPUTS (1) +#define NUM_ANALOG_OUTPUTS (0) + +#define PIN_LED1 (32 + 15) // green +#define LED_BLUE PIN_LED1 // fake for bluefruit library +#define LED_GREEN PIN_LED1 +#define LED_STATE_ON 0 // State when LED is lit + +/* + * Buttons + */ +#define PIN_BUTTON1 (32 + 10) // P1.10, MCU_USER + +/* +No longer populated on PCB +*/ +#define PIN_SERIAL2_RX (-1) +#define PIN_SERIAL2_TX (-1) + +/* + * I2C + */ + +#define WIRE_INTERFACES_COUNT 1 + +// I2C bus 0, routed to HUSB238 USB PD sink controller. +#define PIN_WIRE_SDA (0 + 30) // P0.30, PD_SINK_SDA +#define PIN_WIRE_SCL (0 + 5) // P0.05, PD_SINK_SCL + +/* + * Lora radio + */ +#define USE_SX1262 +#define SX126X_CS (0 + 24) +#define LORA_CS SX126X_CS +#define SX126X_DIO1 (0 + 20) +#define SX126X_BUSY (0 + 17) +#define SX126X_RESET (0 + 25) +#define SX126X_DIO2_AS_RF_SWITCH +#define SX126X_DIO3_TCXO_VOLTAGE 1.8 + +#define USE_KCT8103L_PA_ONLY +#define LORA_KCT8103L_EN (0 + 15) // CSD - KCT8103L chip enable (HIGH=on) +#define LORA_KCT8103L_TX_RX (0 + 16) // TX or bypass control (HIGH=TX, LOW=RX) +#define LORA_PA_POWER LORA_KCT8103L_EN +#define RF_PA_DETECT_PIN (0 + 13) // HIGH=high-power PA, LOW=low-power +#define RF_PA_HIGH_POWER_VALUE HIGH + +/* + * SPI Interfaces + */ +#define SPI_INTERFACES_COUNT 1 + +// For LORA, spi 0 +#define PIN_SPI_MISO (0 + 23) +#define PIN_SPI_MOSI (0 + 22) +#define PIN_SPI_SCK (0 + 19) + +/* + * GPS pins + */ + +#define GPS_L76K + +#define PIN_GPS_RESET (32 + 6) +#define GPS_RESET_MODE LOW +#define PIN_GPS_EN (0 + 7) // P0.07, VGNSS_Ctrl +#define GPS_EN_ACTIVE LOW +#define PERIPHERAL_WARMUP_MS 1000 // Make sure GNSS power is stable before continuing +#define PIN_GPS_STANDBY (32 + 2) // P1.02, WAKE_UP. Low allows sleep, high forces wake. +#define PIN_GPS_PPS (32 + 4) // P1.04, 1PPS +#define GPS_RX_PIN (32 + 5) // P1.05, MCU RX connected to GPS TXD. +#define GPS_TX_PIN (32 + 7) // P1.07, MCU TX connected to GPS RXD. + +#define GPS_THREAD_INTERVAL 50 + +#define PIN_SERIAL1_RX GPS_RX_PIN +#define PIN_SERIAL1_TX GPS_TX_PIN + +// Hardware watchdog +#define HAS_HARDWARE_WATCHDOG +#define HARDWARE_WATCHDOG_DONE (0 + 9) +#define HARDWARE_WATCHDOG_WAKE (0 + 10) +#define HARDWARE_WATCHDOG_TIMEOUT_MS (6 * 60 * 1000) // 6 minute watchdog + +#define SERIAL_PRINT_PORT 0 + +#define ADC_CTRL (0 + 21) // P0.21, ADC_Ctrl +#define ADC_CTRL_ENABLED HIGH +#define BATTERY_PIN (0 + 4) // P0.04, ADC_IN +#define ADC_RESOLUTION 14 + +#define BATTERY_SENSE_RESOLUTION_BITS 12 +#define BATTERY_SENSE_RESOLUTION 4096.0 +#undef AREF_VOLTAGE +#define AREF_VOLTAGE 3.0 +#define VBAT_AR_INTERNAL AR_INTERNAL_3_0 +#define ADC_MULTIPLIER (4.916F) + +// nRF52840 AIN2 is P0.04 on the MeshTower V2 battery divider. +#define BATTERY_LPCOMP_INPUT NRF_LPCOMP_INPUT_2 + +// We have AIN2 with a VBAT divider so AIN2 = VBAT * (100/490) +// We have the device going deep sleep under 3.1V, which is AIN2 = 0.63V +// So we can wake up when VBAT>=VDD is restored to 3.3V, where AIN2 = 0.67V +// Ratio 0.67/3.3 = 0.20, so we can pick a bit higher, 2/8 VDD, which means +// VBAT=4.04V +#define BATTERY_LPCOMP_THRESHOLD NRF_LPCOMP_REF_SUPPLY_2_8 + +#ifdef __cplusplus +} +#endif + +/*---------------------------------------------------------------------------- + * Arduino objects - C++ only + *----------------------------------------------------------------------------*/ + +#endif From 4f0e2dde98f07074ed42c1627b3b05c36283fb7f Mon Sep 17 00:00:00 2001 From: Carlos Valdes Date: Thu, 18 Jun 2026 16:28:10 +0200 Subject: [PATCH 14/86] feat: add Ethernet OTA support for RP2350/W5500 boards (#10136) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add Raspberry Pi Pico 2 + W5500 + E22-900M30S variant Adds community variant for Raspberry Pi Pico 2 (RP2350, 4 MB flash) with external WIZnet W5500 Ethernet module and EBYTE E22-900M30S LoRa module (SX1262, 30 dBm PA, 868/915 MHz). Key details: - LoRa on SPI1: GP10/11/12/13 (SCK/MOSI/MISO/CS), RST=GP15, DIO1=GP14, BUSY=GP2, RXEN=GP3 (held HIGH via SX126X_ANT_SW) - W5500 on SPI0: GP16/17/18/19/20 (MISO/CS/SCK/MOSI/RST) - SX126X_DIO2_AS_RF_SWITCH: DIO2→TXEN bridge on module handles PA - SX126X_DIO3_TCXO_VOLTAGE 1.8: TCXO support via EBYTE_E22 flags - DHCP timeout reduced to 10 s to avoid blocking LoRa startup - GPS on UART1/Serial2: GP8 TX, GP9 RX - Reuses WIZNET_5500_EVB_PICO2 code paths for Ethernet init Co-Authored-By: Claude Sonnet 4.6 * pico2_w5500_e22: rename define and address review feedback Rename WIZNET_5500_EVB_PICO2 to PICO2_W5500_E22 so the variant-specific define matches the variant directory name and isn't confused with an on-board EVB SKU. Review fixes from PR #10135: - Gate the 10 s Ethernet DHCP timeout behind PICO2_W5500_E22 so other Ethernet builds keep the default 60 s behavior; apply the same timeout to reconnectETH() for consistency. - Drop the unused -D EBYTE_E22 flag; EBYTE_E22_900M30S already selects TX_GAIN_LORA / SX126X_MAX_POWER in src/configuration.h. - Rewrite "on-board W5500" comments to describe the external module. - Correct README TX_GAIN_LORA value (7, not 10) and drop the EBYTE_E22 row. * fix(pico2_w5500_e22): drop DEBUG_RP2040_PORT=Serial The arduino-pico framework hooks _write() when DEBUG_RP2040_PORT=Serial is set and dumps raw debug bytes onto USB CDC, corrupting any binary protobuf stream sent through StreamAPI (e.g. `meshtastic --port COMx`). The variant excludes BT and WiFi, so the primary client transport is Ethernet TCP via ethServerAPI — unaffected — but users who configure the node over USB serial would see protobuf decode failures from debug-byte interleaving. Removing the flag restores clean USB CDC. Debug output can still be enabled per-build by adding -D DEBUG_RP2040_PORT=Serial1 to redirect to UART0 instead of USB CDC. * feat: add Ethernet OTA support for RP2350/W5500 boards Adds over-the-air firmware update capability for RP2350-based boards with a WIZnet W5500 Ethernet module (e.g. pico2_w5500_e22). Protocol (MOTA): - SHA256 challenge-response authentication with a configurable PSK (override via USERPREFS_OTA_PSK; default key ships in source) - 12-byte header: magic "MOTA" + firmware size + CRC32 - Firmware received in 1 KB chunks, verified with CRC32, written via Updater (picoOTA), then device reboots to apply - Constant-time hash comparison prevents timing attacks on auth - 30s inactivity timeout + 5s cooldown after failed auth - Response codes 0x00-0x08 map 1:1 to OTAResponse enum Firmware side: - ethOTA.cpp / ethOTA.h: OTA TCP server on port 4243 - ethClient.cpp: wire initEthOTA/ethOTALoop into reconnect loop - main-rp2xx0.cpp: hardware watchdog (8s, paused during debug) - pico2_w5500_e22/platformio.ini: HAS_ETHERNET_OTA flag, filesystem_size bumped to 0.75m for OTA staging Host side: - bin/eth-ota-upload.py: Python uploader with progress and full result-code mapping (matches OTAResponse 0x00-0x08) * style(eth): clang-format ethOTA.cpp per repo .clang-format Reformat to the repo trunk clang-format config (IndentWidth 4, ColumnLimit 130). Resolves the Trunk Check 'Incorrect formatting' failure on PR #10136. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Sonnet 4.6 --- bin/eth-ota-upload.py | 252 +++++++++++++++ src/mesh/eth/ethClient.cpp | 11 + src/mesh/eth/ethOTA.cpp | 293 ++++++++++++++++++ src/mesh/eth/ethOTA.h | 22 ++ src/platform/rp2xx0/main-rp2xx0.cpp | 15 + .../rp2350/diy/pico2_w5500_e22/platformio.ini | 4 + 6 files changed, 597 insertions(+) create mode 100644 bin/eth-ota-upload.py create mode 100644 src/mesh/eth/ethOTA.cpp create mode 100644 src/mesh/eth/ethOTA.h diff --git a/bin/eth-ota-upload.py b/bin/eth-ota-upload.py new file mode 100644 index 000000000..68bc1e2a7 --- /dev/null +++ b/bin/eth-ota-upload.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +""" +Meshtastic Ethernet OTA Upload Tool + +Uploads firmware to RP2350-based Meshtastic devices via Ethernet (W5500). +Compresses firmware with GZIP and sends it over TCP using the MOTA protocol. +Authenticates using SHA256 challenge-response with a pre-shared key (PSK). + +Usage: + python bin/eth-ota-upload.py --host 192.168.1.100 firmware.bin + python bin/eth-ota-upload.py --host 192.168.1.100 --psk mySecretKey firmware.bin + python bin/eth-ota-upload.py --host 192.168.1.100 --psk-hex 6d65736874... firmware.bin +""" + +import argparse +import gzip +import hashlib +import socket +import struct +import sys +import time + +# Default PSK matching the firmware default: "meshtastic_ota_default_psk_v1!!!" +DEFAULT_PSK = b"meshtastic_ota_default_psk_v1!!!" + + +def crc32(data: bytes) -> int: + """Compute CRC32 matching ErriezCRC32 (standard CRC32 with final XOR).""" + import binascii + + return binascii.crc32(data) & 0xFFFFFFFF + + +def load_firmware(path: str) -> bytes: + """Load firmware file, compressing with GZIP if not already compressed.""" + # Reject UF2 files — OTA requires raw .bin firmware + if path.lower().endswith(".uf2"): + bin_path = path.rsplit(".", 1)[0] + ".bin" + print(f"ERROR: UF2 files cannot be used for OTA updates.") + print(f" The Updater/picoOTA expects raw .bin firmware.") + print(f" Try: {bin_path}") + sys.exit(1) + + with open(path, "rb") as f: + data = f.read() + + # Check if already GZIP compressed (magic bytes 1f 8b) + if data[:2] == b"\x1f\x8b": + print(f"Firmware already GZIP compressed: {len(data):,} bytes") + return data + + print(f"Firmware raw size: {len(data):,} bytes") + compressed = gzip.compress(data, compresslevel=9) + ratio = len(compressed) / len(data) * 100 + print(f"GZIP compressed: {len(compressed):,} bytes ({ratio:.1f}%)") + return compressed + + +def authenticate(sock: socket.socket, psk: bytes) -> bool: + """Perform SHA256 challenge-response authentication with the device.""" + # Receive 32-byte nonce from server + nonce = b"" + while len(nonce) < 32: + chunk = sock.recv(32 - len(nonce)) + if not chunk: + print("ERROR: Connection closed during authentication") + return False + nonce += chunk + + # Compute SHA256(nonce || PSK) + h = hashlib.sha256() + h.update(nonce) + h.update(psk) + response = h.digest() + + # Send 32-byte response + sock.sendall(response) + + # Wait for auth result (1 byte) + result = sock.recv(1) + if not result: + print("ERROR: No authentication response") + return False + + if result[0] == 0x06: # ACK + print("Authentication successful.") + return True + elif result[0] == 0x07: # OTA_ERR_AUTH + print("ERROR: Authentication failed — wrong PSK") + return False + else: + print(f"ERROR: Unexpected auth response 0x{result[0]:02X}") + return False + + +def upload_firmware(host: str, port: int, firmware: bytes, psk: bytes, timeout: float) -> bool: + """Upload firmware over TCP using the MOTA protocol with PSK authentication.""" + fw_crc = crc32(firmware) + fw_size = len(firmware) + + print(f"Connecting to {host}:{port}...") + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(timeout) + + try: + sock.connect((host, port)) + print("Connected.") + + # Step 1: Authenticate + print("Authenticating...") + if not authenticate(sock, psk): + return False + + # Step 2: Send 12-byte MOTA header: magic(4) + size(4) + crc32(4) + header = struct.pack("<4sII", b"MOTA", fw_size, fw_crc) + sock.sendall(header) + print(f"Header sent: size={fw_size:,}, CRC32=0x{fw_crc:08X}") + + # Wait for ACK (1 byte) + ack = sock.recv(1) + if not ack or ack[0] != 0x06: + error_codes = { + 0x02: "Size error", + 0x04: "Invalid magic", + 0x05: "Update.begin() failed", + } + code = ack[0] if ack else 0xFF + msg = error_codes.get(code, f"Unknown error 0x{code:02X}") + print(f"ERROR: Server rejected header: {msg}") + return False + + print("Header accepted. Uploading firmware...") + + # Send firmware in 1KB chunks + chunk_size = 1024 + sent = 0 + start_time = time.time() + + while sent < fw_size: + end = min(sent + chunk_size, fw_size) + chunk = firmware[sent:end] + sock.sendall(chunk) + sent = end + + # Progress bar + pct = sent * 100 // fw_size + bar_len = 40 + filled = bar_len * sent // fw_size + bar = "█" * filled + "░" * (bar_len - filled) + elapsed = time.time() - start_time + speed = sent / elapsed if elapsed > 0 else 0 + sys.stdout.write(f"\r [{bar}] {pct:3d}% {sent:,}/{fw_size:,} ({speed/1024:.1f} KB/s)") + sys.stdout.flush() + + elapsed = time.time() - start_time + print(f"\n Transfer complete in {elapsed:.1f}s") + + # Wait for final result (1 byte) + print("Waiting for verification...") + result = sock.recv(1) + if not result: + print("ERROR: No response from device") + return False + + result_codes = { + 0x00: "OK — Update staged, device rebooting", + 0x01: "CRC mismatch", + 0x02: "Size error", + 0x03: "Write error", + 0x04: "Magic mismatch", + 0x05: "Updater.begin() failed", + 0x07: "Auth failed", + 0x08: "Timeout", + } + code = result[0] + msg = result_codes.get(code, f"Unknown result 0x{code:02X}") + + if code == 0x00: + print(f"SUCCESS: {msg}") + return True + else: + print(f"ERROR: {msg}") + return False + + except socket.timeout: + print("ERROR: Connection timed out") + return False + except ConnectionRefusedError: + print(f"ERROR: Connection refused by {host}:{port}") + return False + except OSError as e: + print(f"ERROR: {e}") + return False + finally: + sock.close() + + +def main(): + parser = argparse.ArgumentParser( + description="Upload firmware to Meshtastic RP2350 devices via Ethernet OTA" + ) + parser.add_argument("firmware", help="Path to firmware .bin or .bin.gz file") + parser.add_argument("--host", required=True, help="Device IP address") + parser.add_argument( + "--port", type=int, default=4243, help="OTA port (default: 4243)" + ) + parser.add_argument( + "--timeout", + type=float, + default=60.0, + help="Socket timeout in seconds (default: 60)", + ) + psk_group = parser.add_mutually_exclusive_group() + psk_group.add_argument( + "--psk", + type=str, + help="Pre-shared key as UTF-8 string (default: meshtastic_ota_default_psk_v1!!!)", + ) + psk_group.add_argument( + "--psk-hex", + type=str, + help="Pre-shared key as hex string (e.g., 6d65736874...)", + ) + args = parser.parse_args() + + # Resolve PSK + if args.psk: + psk = args.psk.encode("utf-8") + elif args.psk_hex: + try: + psk = bytes.fromhex(args.psk_hex) + except ValueError: + print("ERROR: Invalid hex string for --psk-hex") + sys.exit(1) + else: + psk = DEFAULT_PSK + + print("Meshtastic Ethernet OTA Upload") + print("=" * 40) + + firmware = load_firmware(args.firmware) + + if upload_firmware(args.host, args.port, firmware, psk, args.timeout): + print("\nDevice is rebooting with new firmware.") + sys.exit(0) + else: + print("\nUpload failed.") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/src/mesh/eth/ethClient.cpp b/src/mesh/eth/ethClient.cpp index a039953f4..003ef8670 100644 --- a/src/mesh/eth/ethClient.cpp +++ b/src/mesh/eth/ethClient.cpp @@ -6,6 +6,9 @@ #include "main.h" #include "mesh/api/ethServerAPI.h" #include "target_specific.h" +#if HAS_ETHERNET && defined(HAS_ETHERNET_OTA) +#include "mesh/eth/ethOTA.h" +#endif #ifdef USE_ARDUINO_ETHERNET #include // arduino-libraries/Ethernet — supports W5100/W5200/W5500 // Shorter DHCP timeout so LoRa startup isn't blocked when no DHCP server is present. @@ -154,6 +157,10 @@ static int32_t reconnectETH() } #endif +#if HAS_ETHERNET && defined(HAS_ETHERNET_OTA) + initEthOTA(); +#endif + ethStartupComplete = true; } } @@ -180,6 +187,10 @@ static int32_t reconnectETH() } #endif +#if HAS_ETHERNET && defined(HAS_ETHERNET_OTA) + ethOTALoop(); +#endif + return 5000; // every 5 seconds } diff --git a/src/mesh/eth/ethOTA.cpp b/src/mesh/eth/ethOTA.cpp new file mode 100644 index 000000000..7519a6112 --- /dev/null +++ b/src/mesh/eth/ethOTA.cpp @@ -0,0 +1,293 @@ +#include "configuration.h" + +#if HAS_ETHERNET && defined(HAS_ETHERNET_OTA) + +#include "ethOTA.h" +#include +#include +#include +#ifdef ARCH_RP2040 +#include +#define FEED_WATCHDOG() watchdog_update() +#else +#define FEED_WATCHDOG() ((void)0) +#endif + +/// Protocol header sent by the upload tool +struct __attribute__((packed)) OTAHeader { + uint8_t magic[4]; // "MOTA" (Meshtastic OTA) + uint32_t firmwareSize; // Size of the firmware payload in bytes (little-endian) + uint32_t crc32; // CRC32 of the entire firmware payload +}; + +/// Response codes sent back to the client +enum OTAResponse : uint8_t { + OTA_OK = 0x00, + OTA_ERR_CRC = 0x01, + OTA_ERR_SIZE = 0x02, + OTA_ERR_WRITE = 0x03, + OTA_ERR_MAGIC = 0x04, + OTA_ERR_BEGIN = 0x05, + OTA_ACK = 0x06, // ACK uses ASCII ACK character + OTA_ERR_AUTH = 0x07, + OTA_ERR_TIMEOUT = 0x08, +}; + +static const uint32_t OTA_TIMEOUT_MS = 30000; // 30s inactivity timeout +static const size_t OTA_CHUNK_SIZE = 1024; // 1KB receive buffer +static const uint32_t OTA_AUTH_COOLDOWN_MS = 5000; // 5s cooldown after failed auth +static const size_t OTA_NONCE_SIZE = 32; +static const size_t OTA_HASH_SIZE = 32; + +// OTA PSK — override via USERPREFS_OTA_PSK in userPrefs.jsonc +// USERPREFS_OTA_PSK is stringified by PlatformIO (wrapped in quotes), so we +// use a char[] and sizeof-1 to exclude the trailing NUL byte from the hash. +#ifdef USERPREFS_OTA_PSK +static const char otaPSKString[] = USERPREFS_OTA_PSK; +static const uint8_t *const otaPSK = reinterpret_cast(otaPSKString); +static const size_t otaPSKSize = sizeof(otaPSKString) - 1; +#else +// Default PSK (CHANGE THIS for production deployments) +static const uint8_t otaPSK[] = {0x6d, 0x65, 0x73, 0x68, 0x74, 0x61, 0x73, 0x74, 0x69, 0x63, 0x5f, 0x6f, 0x74, 0x61, 0x5f, 0x64, + 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x70, 0x73, 0x6b, 0x5f, 0x76, 0x31, 0x21, 0x21, 0x21}; +// = "meshtastic_ota_default_psk_v1!!!" +static const size_t otaPSKSize = sizeof(otaPSK); +#endif + +static EthernetServer *otaServer = nullptr; +static uint32_t lastAuthFailure = 0; + +static bool readExact(EthernetClient &client, uint8_t *buf, size_t len) +{ + size_t received = 0; + uint32_t lastActivity = millis(); + + while (received < len) { + if (!client.connected()) { + return false; + } + int avail = client.available(); + if (avail > 0) { + size_t toRead = min((size_t)avail, len - received); + size_t got = client.read(buf + received, toRead); + received += got; + lastActivity = millis(); + } else { + if (millis() - lastActivity > OTA_TIMEOUT_MS) { + return false; + } + delay(1); + } + FEED_WATCHDOG(); + } + return true; +} + +/// Compute SHA256(nonce || psk) for challenge-response authentication +static void computeAuthHash(const uint8_t *nonce, size_t nonceLen, const uint8_t *psk, size_t pskLen, uint8_t *hashOut) +{ + SHA256 sha; + sha.reset(); + sha.update(nonce, nonceLen); + sha.update(psk, pskLen); + sha.finalize(hashOut, OTA_HASH_SIZE); +} + +/// Challenge-response authentication. Returns true if client is authenticated. +static bool authenticateClient(EthernetClient &client) +{ + // Rate-limit after failed auth — close silently so the error byte is not + // misinterpreted as part of the nonce by a re-trying client. + if (lastAuthFailure != 0 && (millis() - lastAuthFailure) < OTA_AUTH_COOLDOWN_MS) { + LOG_WARN("ETH OTA: Auth cooldown active, rejecting connection"); + client.stop(); + return false; + } + + // Generate random nonce + uint8_t nonce[OTA_NONCE_SIZE]; + for (size_t i = 0; i < OTA_NONCE_SIZE; i += 4) { + uint32_t r = random(); + size_t remaining = OTA_NONCE_SIZE - i; + memcpy(nonce + i, &r, min((size_t)4, remaining)); + } + + // Send nonce to client + client.write(nonce, OTA_NONCE_SIZE); + + // Read client's response: SHA256(nonce || PSK) + uint8_t clientHash[OTA_HASH_SIZE]; + if (!readExact(client, clientHash, OTA_HASH_SIZE)) { + LOG_WARN("ETH OTA: Timeout reading auth response"); + lastAuthFailure = millis(); + return false; + } + + // Compute expected hash + uint8_t expectedHash[OTA_HASH_SIZE]; + computeAuthHash(nonce, OTA_NONCE_SIZE, otaPSK, otaPSKSize, expectedHash); + + // Constant-time comparison to prevent timing attacks + uint8_t diff = 0; + for (size_t i = 0; i < OTA_HASH_SIZE; i++) { + diff |= clientHash[i] ^ expectedHash[i]; + } + + if (diff != 0) { + LOG_WARN("ETH OTA: Authentication failed"); + client.write(OTA_ERR_AUTH); + lastAuthFailure = millis(); + return false; + } + + // Auth success — send ACK + client.write(OTA_ACK); + LOG_INFO("ETH OTA: Authentication successful"); + return true; +} + +static void handleOTAClient(EthernetClient &client) +{ + LOG_INFO("ETH OTA: Client connected from %u.%u.%u.%u", client.remoteIP()[0], client.remoteIP()[1], client.remoteIP()[2], + client.remoteIP()[3]); + + // Step 1: Challenge-response authentication + if (!authenticateClient(client)) { + return; + } + + // Step 2: Read 12-byte header + OTAHeader hdr; + if (!readExact(client, (uint8_t *)&hdr, sizeof(hdr))) { + LOG_WARN("ETH OTA: Timeout reading header"); + return; + } + + // Validate magic + if (memcmp(hdr.magic, "MOTA", 4) != 0) { + LOG_WARN("ETH OTA: Invalid magic"); + client.write(OTA_ERR_MAGIC); + return; + } + + LOG_INFO("ETH OTA: Firmware size=%u, CRC32=0x%08X", hdr.firmwareSize, hdr.crc32); + + // Sanity check on size (must be > 0 and fit in LittleFS) + if (hdr.firmwareSize == 0 || hdr.firmwareSize > 1024 * 1024) { + LOG_WARN("ETH OTA: Invalid firmware size"); + client.write(OTA_ERR_SIZE); + return; + } + + // Begin the update — this opens firmware.bin on LittleFS + if (!Update.begin(hdr.firmwareSize)) { + LOG_ERROR("ETH OTA: Update.begin() failed, error=%u", Update.getError()); + client.write(OTA_ERR_BEGIN); + return; + } + + // ACK the header — client can start sending firmware data + client.write(OTA_ACK); + + // Receive firmware in chunks + uint8_t buf[OTA_CHUNK_SIZE]; + size_t remaining = hdr.firmwareSize; + uint32_t crc = CRC32_INITIAL; + uint32_t lastActivity = millis(); + size_t totalReceived = 0; + + while (remaining > 0) { + if (!client.connected()) { + LOG_WARN("ETH OTA: Client disconnected during transfer"); + Update.end(false); + return; + } + + int avail = client.available(); + if (avail <= 0) { + if (millis() - lastActivity > OTA_TIMEOUT_MS) { + LOG_WARN("ETH OTA: Timeout during transfer (%u/%u bytes)", totalReceived, hdr.firmwareSize); + client.write(OTA_ERR_TIMEOUT); + Update.end(false); + return; + } + delay(1); + FEED_WATCHDOG(); + continue; + } + + size_t toRead = min((size_t)avail, min(remaining, sizeof(buf))); + size_t got = client.read(buf, toRead); + if (got == 0) + continue; + + // Write to Updater (LittleFS firmware.bin) + size_t written = Update.write(buf, got); + if (written != got) { + LOG_ERROR("ETH OTA: Write failed (wrote %u of %u), error=%u", written, got, Update.getError()); + client.write(OTA_ERR_WRITE); + Update.end(false); + return; + } + + crc = crc32Update(buf, got, crc); + remaining -= got; + totalReceived += got; + lastActivity = millis(); + FEED_WATCHDOG(); + + // Progress log every ~10% + if (totalReceived % (hdr.firmwareSize / 10 + 1) < got) { + LOG_INFO("ETH OTA: %u%% (%u/%u bytes)", (uint32_t)(100ULL * totalReceived / hdr.firmwareSize), totalReceived, + hdr.firmwareSize); + } + } + + // Verify CRC32 + uint32_t computedCRC = crc32Final(crc); + if (computedCRC != hdr.crc32) { + LOG_ERROR("ETH OTA: CRC mismatch (expected=0x%08X, computed=0x%08X)", hdr.crc32, computedCRC); + client.write(OTA_ERR_CRC); + Update.end(false); + return; + } + + // Finalize — this calls picoOTA.commit() which stages the update for the + // bootloader + if (!Update.end(true)) { + LOG_ERROR("ETH OTA: Update.end() failed, error=%u", Update.getError()); + client.write(OTA_ERR_WRITE); + return; + } + + LOG_INFO("ETH OTA: Update staged successfully (%u bytes). Rebooting...", hdr.firmwareSize); + client.write(OTA_OK); + client.flush(); + delay(500); + + // Reboot — the built-in bootloader will apply the update from LittleFS + rp2040.reboot(); +} + +void initEthOTA() +{ + if (!otaServer) { + otaServer = new EthernetServer(ETH_OTA_PORT); + otaServer->begin(); + LOG_INFO("ETH OTA: Server listening on TCP port %d", ETH_OTA_PORT); + } +} + +void ethOTALoop() +{ + if (!otaServer) + return; + + EthernetClient client = otaServer->accept(); + if (client) { + handleOTAClient(client); + client.stop(); + } +} + +#endif // HAS_ETHERNET && HAS_ETHERNET_OTA diff --git a/src/mesh/eth/ethOTA.h b/src/mesh/eth/ethOTA.h new file mode 100644 index 000000000..d2659958e --- /dev/null +++ b/src/mesh/eth/ethOTA.h @@ -0,0 +1,22 @@ +#pragma once + +#include "configuration.h" + +#if HAS_ETHERNET && defined(HAS_ETHERNET_OTA) + +#ifdef USE_ARDUINO_ETHERNET +#include +#else +#include +#endif + +#define ETH_OTA_PORT 4243 + +/// Initialize the Ethernet OTA server (call after Ethernet is connected) +void initEthOTA(); + +/// Poll for incoming OTA connections (call periodically from ethClient +/// reconnect loop) +void ethOTALoop(); + +#endif // HAS_ETHERNET && HAS_ETHERNET_OTA diff --git a/src/platform/rp2xx0/main-rp2xx0.cpp b/src/platform/rp2xx0/main-rp2xx0.cpp index e59b0a9cd..ee50f4fb1 100644 --- a/src/platform/rp2xx0/main-rp2xx0.cpp +++ b/src/platform/rp2xx0/main-rp2xx0.cpp @@ -3,6 +3,7 @@ #include "hardware/xosc.h" #include #include +#include #include #include @@ -99,6 +100,10 @@ void getMacAddr(uint8_t *dmac) void rp2040Setup() { + if (watchdog_caused_reboot()) { + LOG_WARN("Rebooted by watchdog!"); + } + /* Sets a random seed to make sure we get different random numbers on each boot. */ uint32_t seed = 0; if (!HardwareRNG::seed(seed)) { @@ -128,6 +133,16 @@ void rp2040Setup() #endif } +void rp2040Loop() +{ + static bool watchdog_running = false; + if (!watchdog_running) { + watchdog_enable(8000, true); // 8s timeout; pauses during debug + watchdog_running = true; + } + watchdog_update(); +} + void enterDfuMode() { reset_usb_boot(0, 0); diff --git a/variants/rp2350/diy/pico2_w5500_e22/platformio.ini b/variants/rp2350/diy/pico2_w5500_e22/platformio.ini index 379dd5910..b8535dda5 100644 --- a/variants/rp2350/diy/pico2_w5500_e22/platformio.ini +++ b/variants/rp2350/diy/pico2_w5500_e22/platformio.ini @@ -4,12 +4,16 @@ board = rpipico2 board_level = community upload_protocol = picotool +# Increase LittleFS from 0.5m to 0.75m so GZIP firmware (~614KB) fits for OTA staging +board_build.filesystem_size = 0.75m + build_flags = ${rp2350_base.build_flags} -ULED_BUILTIN # avoid "LED_BUILTIN redefined" warnings from framework common.h -I variants/rp2350/diy/pico2_w5500_e22 -D HW_SPI1_DEVICE -D EBYTE_E22_900M30S # selects the EBYTE E22-900M30S module config, including TCXO voltage support and TX gain / max power settings + -D HAS_ETHERNET_OTA # enable Ethernet OTA firmware update server on port 4243 # Re-enable Ethernet and API source paths excluded in rp2350_base build_src_filter = ${rp2350_base.build_src_filter} + + + From 9358b2c54953b0f0a8e587cbb2a2f931f9a1b26f Mon Sep 17 00:00:00 2001 From: "Ethac.chen" Date: Fri, 19 Jun 2026 05:29:54 +0800 Subject: [PATCH 15/86] fix(esp32): skip RTC timer wake on user shutdown; TAP V2 OCV and partition (#10739) * fix(esp32): skip RTC timer wake on user shutdown Do not arm esp_sleep_enable_timer_wakeup when msecToWake is portMAX_DELAY (UI shutdown), matching nRF52 system_off semantics. fix(rak_wismesh_tap_v2): Tag OCV curve and 16MB partition Add OCV_ARRAY matching WisMesh Tag for accurate SOC. Use 16MB flash partition scheme for TAP V2 hardware. Co-authored-by: Cursor * Trunkt --------- Co-authored-by: Cursor Co-authored-by: Ben Meadors --- src/platform/esp32/main-esp32.cpp | 7 +++++-- variants/esp32s3/rak_wismesh_tap_v2/platformio.ini | 4 ++-- variants/esp32s3/rak_wismesh_tap_v2/variant.h | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/platform/esp32/main-esp32.cpp b/src/platform/esp32/main-esp32.cpp index 69b69ff68..bcd618c7e 100644 --- a/src/platform/esp32/main-esp32.cpp +++ b/src/platform/esp32/main-esp32.cpp @@ -15,6 +15,7 @@ #endif #include "esp_mac.h" +#include "freertosinc.h" #include "meshUtils.h" #include "sleep.h" #include "soc/rtc.h" @@ -276,6 +277,8 @@ void cpuDeepSleep(uint32_t msecToWake) esp_sleep_pd_config(ESP_PD_DOMAIN_RTC_PERIPH, ESP_PD_OPTION_ON); #endif - esp_sleep_enable_timer_wakeup(msecToWake * 1000ULL); // call expects usecs - esp_deep_sleep_start(); // TBD mA sleep current (battery) + // User shutdown (DELAY_FOREVER / portMAX_DELAY): no RTC timer — align with nRF52 system_off semantics. + if (msecToWake != portMAX_DELAY) + esp_sleep_enable_timer_wakeup(msecToWake * 1000ULL); // call expects usecs + esp_deep_sleep_start(); } diff --git a/variants/esp32s3/rak_wismesh_tap_v2/platformio.ini b/variants/esp32s3/rak_wismesh_tap_v2/platformio.ini index be964cc6d..66093d033 100644 --- a/variants/esp32s3/rak_wismesh_tap_v2/platformio.ini +++ b/variants/esp32s3/rak_wismesh_tap_v2/platformio.ini @@ -19,14 +19,14 @@ custom_meshtastic_support_level = 1 custom_meshtastic_display_name = RAK WisMesh Tap V2 custom_meshtastic_images = rak-wismesh-tap-v2.svg custom_meshtastic_tags = RAK -custom_meshtastic_partition_scheme = 8MB +custom_meshtastic_partition_scheme = 16MB custom_meshtastic_has_mui = true extends = esp32s3_base board = wiscore_rak3312 board_check = true upload_protocol = esptool -board_build.partitions = default_8MB.csv +board_build.partitions = default_16MB.csv build_flags = ${esp32s3_base.build_flags} diff --git a/variants/esp32s3/rak_wismesh_tap_v2/variant.h b/variants/esp32s3/rak_wismesh_tap_v2/variant.h index f8672edac..a169a1e8a 100644 --- a/variants/esp32s3/rak_wismesh_tap_v2/variant.h +++ b/variants/esp32s3/rak_wismesh_tap_v2/variant.h @@ -53,6 +53,7 @@ #define BATTERY_PIN 1 #define ADC_CHANNEL ADC_CHANNEL_0 #define ADC_MULTIPLIER 1.667 +#define OCV_ARRAY 4160, 4020, 3940, 3870, 3810, 3760, 3740, 3720, 3680, 3620, 2990 #define PIN_BUZZER 38 From f2f23f89782ac1d6eee73e0bb76979440c8b8777 Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Fri, 19 Jun 2026 12:21:05 -0500 Subject: [PATCH 16/86] NimBLE params overhaul and try-fix for incompatible bond cleanup (#10741) * NimBLE params overhaul and try-fix for incompatible bond cleanup * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Address PR review: remove dead clearNVS(), defer bond-purge log below init - Delete unused clearNVS() (no callers; should have been removed in #10264). - Move purgeIncompatibleBleBonds() after the "Init the NimBLE" log so bond-cleanup output doesn't appear to precede module init; it still runs before BLEDevice::init() reads the store. * Update MAX_SATELLITE_NODES and WARM_NODE_COUNT definitions for ESP32-S3 PSRAM support --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/mesh/mesh-pb-constants.h | 21 ++++--- src/nimble/NimbleBluetooth.cpp | 107 +++++++++++++++++++++++++++++--- variants/esp32/esp32-common.ini | 18 +++++- 3 files changed, 126 insertions(+), 20 deletions(-) diff --git a/src/mesh/mesh-pb-constants.h b/src/mesh/mesh-pb-constants.h index c7edb98a3..6f9f4bc91 100644 --- a/src/mesh/mesh-pb-constants.h +++ b/src/mesh/mesh-pb-constants.h @@ -105,14 +105,17 @@ static inline int get_max_num_nodes() /// Per-map cap (position/telemetry/environment/status): only the freshest /// MAX_SATELLITE_NODES nodes keep satellite payloads, the rest just the -/// NodeInfoLite header. RAM-bound (the maps are internal-SRAM, not PSRAM), so -/// flash-rich hosts get a cap >= their hot store (satellites for every node, as -/// before the cap existed) while constrained parts stay at 40. +/// NodeInfoLite header. RAM-bound: the four maps live in internal SRAM (not +/// PSRAM). PSRAM-equipped ESP32-S3 (and native) keep the full 250; other ESP32 +/// (no-PSRAM, incl. S3) get 80 -- ~32 KB worst case, affordable now the warm +/// tier is trimmed; nRF52840 and other tight parts stay at 40. #ifndef MAX_SATELLITE_NODES -#if defined(CONFIG_IDF_TARGET_ESP32S3) || defined(ARCH_PORTDUINO) -#define MAX_SATELLITE_NODES 250 +#if defined(ARCH_PORTDUINO) || (defined(CONFIG_IDF_TARGET_ESP32S3) && defined(BOARD_HAS_PSRAM)) +#define MAX_SATELLITE_NODES 250 // native / PSRAM-equipped ESP32-S3 +#elif defined(ARCH_ESP32) +#define MAX_SATELLITE_NODES 80 // no-PSRAM ESP32 (incl. ESP32-S3) #else -#define MAX_SATELLITE_NODES 40 // nRF52840 (28 KB LittleFS) and generic ESP32 +#define MAX_SATELLITE_NODES 40 // nRF52840 (28 KB LittleFS) and other constrained parts #endif // platform #endif // MAX_SATELLITE_NODES @@ -127,9 +130,11 @@ static inline int get_max_num_nodes() // architecture.h via configuration.h) isn't defined this early in every include // chain. Backed by the raw-flash ring below LittleFS — see WarmNodeStore.h. #define WARM_NODE_COUNT 200 -#elif defined(CONFIG_IDF_TARGET_ESP32S3) -#define WARM_NODE_COUNT 2000 // PSRAM-backed when available; warm.dat ~80 KB +#elif defined(CONFIG_IDF_TARGET_ESP32S3) && defined(BOARD_HAS_PSRAM) +#define WARM_NODE_COUNT 2000 // ESP32-S3 with PSRAM (external); warm.dat ~80 KB #else +// generic ESP32 and no-PSRAM ESP32-S3: ~12.5 KB in internal heap (calloc fallback in +// WarmNodeStore), leaving room for the BLE controller. PSRAM-equipped S3 takes the 2000 case above. #define WARM_NODE_COUNT 320 #endif // platform #endif // WARM_NODE_COUNT diff --git a/src/nimble/NimbleBluetooth.cpp b/src/nimble/NimbleBluetooth.cpp index 5722a6691..3ea0aab31 100644 --- a/src/nimble/NimbleBluetooth.cpp +++ b/src/nimble/NimbleBluetooth.cpp @@ -21,7 +21,12 @@ #include "PowerStatus.h" #include "host/ble_gap.h" +#include "host/ble_hs.h" #include "host/ble_store.h" +#ifdef ARCH_ESP32 +#include +#include +#endif namespace { @@ -30,6 +35,56 @@ constexpr uint16_t kPreferredBleTxOctets = 251; constexpr uint16_t kPreferredBleTxTimeUs = (kPreferredBleTxOctets + 14) * 8; } // namespace +#ifdef ARCH_ESP32 +// Discard NimBLE bonds left in an incompatible on-disk format. The ESP-IDF/NimBLE upgrade changed +// the length of the fixed-size bond records (ble_store_value_sec), so the new host rejects every +// old record on each boot ("NVS data size mismatch for obj_type 1 ...") with no auto-recovery -- +// pairing stays broken until a factory reset. Wipe the bond namespace once when a stored record's +// size differs from this build's struct; a same-size store is left untouched, so this never loops. +// Adapted from https://github.com/h2zero/NimBLE-Arduino/issues/740 +static void purgeIncompatibleBleBonds() +{ + esp_err_t initErr = nvs_flash_init(); + if (initErr != ESP_OK) { + LOG_WARN("purgeIncompatibleBleBonds: nvs_flash_init failed, err=%d", (int)initErr); + return; // NVS should already be up; if not, nothing safe to do here + } + + nvs_handle_t handle = 0; + esp_err_t err = nvs_open("nimble_bond", NVS_READWRITE, &handle); + if (err == ESP_ERR_NVS_NOT_FOUND) { + return; // no bonds stored yet + } + if (err != ESP_OK) { + LOG_ERROR("nimble_bond open failed, err=%d", err); + return; + } + + // Probe the first record of each fixed-size object type (bonds are written from index 1); a + // stored size differing from this build's struct means the store predates a format change. + size_t sz = 0; + bool mismatch = (nvs_get_blob(handle, "our_sec_1", nullptr, &sz) == ESP_OK && sz != sizeof(struct ble_store_value_sec)) || + (nvs_get_blob(handle, "peer_sec_1", nullptr, &sz) == ESP_OK && sz != sizeof(struct ble_store_value_sec)) || + (nvs_get_blob(handle, "cccd_sec_1", nullptr, &sz) == ESP_OK && sz != sizeof(struct ble_store_value_cccd)); + + bool wiped = false; + if (mismatch) { + LOG_WARN("Wiping incompatible NimBLE bonds (on-disk format changed)"); + wiped = nvs_erase_all(handle) == ESP_OK && nvs_commit(handle) == ESP_OK; + if (!wiped) { + LOG_ERROR("Failed to erase nimble_bond namespace"); + } + } + + nvs_close(handle); + + if (wiped) { + LOG_INFO("Restarting after NimBLE bond cleanup"); + ESP.restart(); + } +} +#endif + // Debugging options: careful, they slow things down quite a bit! // #define DEBUG_NIMBLE_ON_READ_TIMING // uncomment to time onRead duration // #define DEBUG_NIMBLE_ON_WRITE_TIMING // uncomment to time onWrite duration @@ -47,6 +102,11 @@ BLEServer *bleServer; static bool passkeyShowing; static std::atomic nimbleBluetoothConnHandle{BLE_HS_CONN_HANDLE_NONE}; // BLE_HS_CONN_HANDLE_NONE means "no connection" +// Set by onDisconnect to defer (re)starting advertising to the main task. A stale-bond reconnect +// triggers a MIC failure + NimBLE host reset; re-entering ble_gap_adv_* from the disconnect +// callback while the host is mid-reset crashes (LoadProhibited), so the main task does it instead. +static std::atomic pendingStartAdvertising{false}; + static void clearPairingDisplay() { if (!passkeyShowing) { @@ -155,6 +215,21 @@ class BluetoothPhoneAPI : public PhoneAPI, public concurrency::OSThread protected: virtual int32_t runOnce() override { + // Service a deferred advertising restart from onDisconnect, gated on ble_hs_synced() so we + // never re-enter the GAP API while the host is still mid-reset. + if (pendingStartAdvertising) { + if (checkIsConnected()) { + pendingStartAdvertising = false; // a new physical connection beat us to it; nothing to do + } else if (ble_hs_synced()) { + pendingStartAdvertising = false; + if (nimbleBluetooth) { + nimbleBluetooth->startAdvertising(); + } + } else { + return 200; // host still re-syncing after a reset; retry shortly + } + } + while (runOnceHasWorkToDo()) { /* PROCESS fromPhoneQueue BEFORE toPhoneQueue: @@ -592,6 +667,14 @@ class NimbleBluetoothSecurityCallback : public BLESecurityCallbacks } void onAuthenticationComplete(ble_gap_conn_desc *desc) override { + // Called on every BLE_GAP_EVENT_ENC_CHANGE, success or failure. A stale-bond reconnect + // yields a *failed* encryption change here -- don't latch a connected/authenticated state + // on a link that is actually being torn down. + if (desc == nullptr || !desc->sec_state.encrypted) { + LOG_WARN("BLE encryption change without an encrypted link; ignoring"); + return; + } + LOG_INFO("BLE authentication complete"); meshtastic::BluetoothStatus newStatus(meshtastic::BluetoothStatus::ConnectionState::CONNECTED); @@ -667,7 +750,13 @@ class NimbleBluetoothServerCallback : public BLEServerCallbacks nimbleBluetoothConnHandle = BLE_HS_CONN_HANDLE_NONE; - ble->startAdvertising(); + // Defer the advertising restart to runOnce (see pendingStartAdvertising): calling + // startAdvertising() here would crash if this disconnect was a host reset. + pendingStartAdvertising = true; + if (bluetoothPhoneAPI) { + bluetoothPhoneAPI->setIntervalFromNow(0); + } + concurrency::mainDelay.interrupt(); // wake the main loop to service the restart } }; @@ -761,6 +850,12 @@ void NimbleBluetooth::setup() LOG_INFO("Init the NimBLE bluetooth module"); +#ifdef ARCH_ESP32 + // Runs before BLEDevice::init() reads the bond store, but logs after the "Init" line above so + // any bond-cleanup output doesn't appear to precede the module init. + purgeIncompatibleBleBonds(); // wipe bonds left in an incompatible on-disk format (post-upgrade) +#endif + BLEDevice::init(getDeviceName()); BLEDevice::setPower(ESP_PWR_LVL_P9); @@ -889,6 +984,7 @@ void updateBatteryLevel(uint8_t level) void NimbleBluetooth::clearBonds() { LOG_INFO("Clearing bluetooth bonds!"); + ble_store_util_delete_all(BLE_STORE_OBJ_TYPE_OUR_SEC, nullptr); ble_store_util_delete_all(BLE_STORE_OBJ_TYPE_PEER_SEC, nullptr); ble_store_util_delete_all(BLE_STORE_OBJ_TYPE_CCCD, nullptr); } @@ -901,13 +997,4 @@ void NimbleBluetooth::sendLog(const uint8_t *logMessage, size_t length) logRadioCharacteristic->setValue(logMessage, length); logRadioCharacteristic->notify(); } - -void clearNVS() -{ - ble_store_util_delete_all(BLE_STORE_OBJ_TYPE_PEER_SEC, nullptr); - ble_store_util_delete_all(BLE_STORE_OBJ_TYPE_CCCD, nullptr); -#ifdef ARCH_ESP32 - ESP.restart(); -#endif -} #endif diff --git a/variants/esp32/esp32-common.ini b/variants/esp32/esp32-common.ini index 75aa8fc63..fac7cc911 100644 --- a/variants/esp32/esp32-common.ini +++ b/variants/esp32/esp32-common.ini @@ -276,9 +276,23 @@ custom_sdkconfig = CONFIG_BT_NIMBLE_ROLE_CENTRAL=n CONFIG_BT_NIMBLE_ROLE_OBSERVER=n CONFIG_BT_CONTROLLER_ENABLED=y - CONFIG_BT_NIMBLE_HOST_TASK_STACK_SIZE=8192 - CONFIG_BT_NIMBLE_MAX_CCCDS=20 + # BLE RAM right-sizing for a single-phone peripheral. IDF-5.5/Arduino-3.x raised RAM use to where + # NimBLE bring-up no longer had enough contiguous heap (host task fails to allocate -> host never + # syncs -> BLEDevice::init() hangs; GATT/advertising then OOM). The node only ever has one + # connection and never scans, so trimming these over-provisioned controller/host buffers frees + # the heap. Keep the host-task stack at the IDF default 5120 (do NOT lower to 4096 -- #2618 raised + # it because 4096 overflows); the prior 8192 was an over-allocation that starved advertising. + CONFIG_BT_NIMBLE_HOST_TASK_STACK_SIZE=5120 + CONFIG_BT_NIMBLE_MAX_CCCDS=8 CONFIG_BT_NIMBLE_MAX_BONDS=6 + CONFIG_BT_NIMBLE_MAX_CONNECTIONS=1 + CONFIG_BT_CTRL_BLE_MAX_ACT=2 + CONFIG_BT_CTRL_SCAN_DUPL_CACHE_SIZE=10 + CONFIG_BT_NIMBLE_WHITELIST_SIZE=1 + CONFIG_BT_NIMBLE_MSYS_1_BLOCK_COUNT=8 + CONFIG_BT_NIMBLE_MSYS_2_BLOCK_COUNT=8 + CONFIG_BT_NIMBLE_TRANSPORT_ACL_FROM_LL_COUNT=8 + CONFIG_BT_NIMBLE_TRANSPORT_EVT_COUNT=12 CONFIG_BT_NIMBLE_ENABLE_PERIODIC_SYNC=n CONFIG_BT_NIMBLE_ENABLE_PERIODIC_ADV=n CONFIG_BT_NIMBLE_EXT_SCAN=n From ca7d82629d6a3750b72e6dfd4878423430ad6b76 Mon Sep 17 00:00:00 2001 From: Jonathan Bennett Date: Fri, 19 Jun 2026 15:50:31 -0500 Subject: [PATCH 17/86] Native sensors (#10748) * Enable Sensirion libraries on native * Bump platform-native to get bug fix --- platformio.ini | 20 ++++++++++---------- variants/native/portduino.ini | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/platformio.ini b/platformio.ini index 99fc35d95..d9c685e8b 100644 --- a/platformio.ini +++ b/platformio.ini @@ -200,6 +200,16 @@ lib_deps = https://github.com/adafruit/Adafruit_TSL2561/archive/refs/tags/1.1.3.zip # renovate: datasource=github-tags depName=BH1750_WE packageName=wollewald/BH1750_WE https://github.com/wollewald/BH1750_WE/archive/refs/tags/1.1.10.zip + # renovate: datasource=github-tags depName=Sensirion Core packageName=sensirion/arduino-core + https://github.com/Sensirion/arduino-core/archive/refs/tags/0.7.3.zip + # renovate: datasource=github-tags depName=Sensirion I2C SCD4x packageName=sensirion/arduino-i2c-scd4x + https://github.com/Sensirion/arduino-i2c-scd4x/archive/refs/tags/1.1.0.zip + # renovate: datasource=github-tags depName=Sensirion I2C SFA3x packageName=sensirion/arduino-i2c-sfa3x + https://github.com/Sensirion/arduino-i2c-sfa3x/archive/refs/tags/1.0.0.zip + # renovate: datasource=github-tags depName=Sensirion I2C SCD30 packageName=sensirion/arduino-i2c-scd30 + https://github.com/Sensirion/arduino-i2c-scd30/archive/refs/tags/1.0.0.zip + # renovate: datasource=github-tags depName=arduino-sht packageName=sensirion/arduino-sht + https://github.com/Sensirion/arduino-sht/archive/refs/tags/v1.2.6.zip ; Common environmental sensor libraries (not included in native / portduino) [environmental_extra_common] @@ -218,16 +228,6 @@ lib_deps = closedcube/ClosedCube OPT3001@1.1.2 # renovate: datasource=git-refs depName=meshtastic-DFRobot_LarkWeatherStation packageName=https://github.com/meshtastic/DFRobot_LarkWeatherStation gitBranch=master https://github.com/meshtastic/DFRobot_LarkWeatherStation/archive/4de3a9cadef0f6a5220a8a906cf9775b02b0040d.zip - # renovate: datasource=github-tags depName=Sensirion Core packageName=sensirion/arduino-core - https://github.com/Sensirion/arduino-core/archive/refs/tags/0.7.3.zip - # renovate: datasource=github-tags depName=Sensirion I2C SCD4x packageName=sensirion/arduino-i2c-scd4x - https://github.com/Sensirion/arduino-i2c-scd4x/archive/refs/tags/1.1.0.zip - # renovate: datasource=github-tags depName=Sensirion I2C SFA3x packageName=sensirion/arduino-i2c-sfa3x - https://github.com/Sensirion/arduino-i2c-sfa3x/archive/refs/tags/1.0.0.zip - # renovate: datasource=github-tags depName=Sensirion I2C SCD30 packageName=sensirion/arduino-i2c-scd30 - https://github.com/Sensirion/arduino-i2c-scd30/archive/refs/tags/1.0.0.zip - # renovate: datasource=github-tags depName=arduino-sht packageName=sensirion/arduino-sht - https://github.com/Sensirion/arduino-sht/archive/refs/tags/v1.2.6.zip ; Environmental sensors with BSEC2 (Bosch proprietary IAQ) [environmental_extra] diff --git a/variants/native/portduino.ini b/variants/native/portduino.ini index 699286e30..3ba852387 100644 --- a/variants/native/portduino.ini +++ b/variants/native/portduino.ini @@ -2,7 +2,7 @@ [portduino_base] platform = # renovate: datasource=git-refs depName=platform-native packageName=https://github.com/meshtastic/platform-native gitBranch=develop - https://github.com/meshtastic/platform-native/archive/cab4b21d902973e43c938dab3cf4844ba02547ec.zip + https://github.com/meshtastic/platform-native/archive/61067ac3774e4fe27aa9762c72cebea507f116c8.zip framework = arduino build_src_filter = From 22072c5f4b106a338e59ffe3c13b8d026e737dac Mon Sep 17 00:00:00 2001 From: Tom <116762865+NomDeTom@users.noreply.github.com> Date: Sat, 20 Jun 2026 01:52:58 +0100 Subject: [PATCH 18/86] Pr1.5 tmm nexthop (#10745) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * TrafficManagement: flat unified cache + persistent next-hop overflow store Reworks the TrafficManagementModule cache layer (policing behaviour unchanged from upstream) and adds a routing-hint overflow store: - Flatten the ring: replace the cuckoo-hashed unified cache and the bucketed PSRAM NodeInfo index with plain flat arrays + linear scan (same idiom as WarmNodeStore). At LoRa packet rates an O(n) scan of the cache is negligible, and it removes a large amount of hashing/displacement complexity. The cache entry is 11 B; timestamps use a uniform +1 presence-offset so a 0 byte always means "empty" across every sub-store. Adds rebaseEpoch() so cached state survives the ~19 h relative-timestamp horizon instead of being flushed. - Next-hop overflow cache: setNextHop/getNextHopHint store a confirmed last-byte relay for a destination, written only from NextHopRouter's ACK-confirmed decision (and mirrored from TraceRoute). NextHopRouter::getNextHop falls back to this cache when the hot NodeDB has no hint, so DMs/relays to long-tail nodes keep routing after the node ages out of NodeInfoLite. - Persistence: preloadNextHopsFromNodeDB warm-starts the cache from persisted NodeInfoLite hints on first maintenance pass; next_hop entries are kept alive across the maintenance sweep (no TTL) and never clobbered by a stale preload. All packet-policing logic (rate limit, position dedup, unknown-packet drop, NodeInfo direct response, hop exhaustion) is the existing upstream behaviour, untouched. HAS_TRAFFIC_MANAGEMENT defaults on so the module is compiled in. (see note). Tests: upstream policing suite now actually runs (adds the MeshTypes.h include that gates HAS_TRAFFIC_MANAGEMENT) plus 4 next-hop tests. Role-aware throttles, politeness, precision clamp, port-interval and mesh-radius gating — and the rate-limit >255 saturation fix — are deferred to the advanced-TMM branch. Note: default dedup movement grid moves to ~91m, which also means 1.5km required to end up with the same signature position - coarser and therefore further than before. Co-Authored-By: Claude Opus 4.8 (1M context) * TrafficManagement: fix cppcheck constVariablePointer warning `node` in preloadNextHopsFromNodeDB() is never written through — mark it const to satisfy cppcheck's constVariablePointer check in CI. Co-Authored-By: Claude Sonnet 4.6 * Add multi-hop NextHop recovery tests and unit tests for routing reliability - Introduced a new test suite for multi-hop NextHop directed-message delivery and relay recovery in `test_nexthop_multihop_recovery.py`. This includes tests for end-to-end delivery and recovery after relay drop. - Implemented unit tests in `test_main.cpp` for NextHop routing reliability mitigations, covering: - M1: Ambiguity-aware last-byte resolution. - M2: NextHopRouter's strict-neighbor gate and hop limit checks. - M3: Route-health freshness and failure decay. - Enhanced mock classes to facilitate controlled testing of node behaviors and routing logic. * grafting fixed * Address Copilot review for PR #10735 (NextHop improvements) - docs/nexthop-routing-reliability.md: update status from "no code changes yet" to reflect that mitigations and tests are implemented RAM pressure and MIGRATION_VERBOSE concerns addressed upstream in PR2.5 (per-platform TRAFFIC_MANAGEMENT_CACHE_SIZE) and PR2 (verbose default=0) respectively; (0,0) sentinel fixed in PR2.5. Co-Authored-By: Claude Sonnet 4.6 * CI: fix cppcheck constVariablePointer and test include path - NextHopRouter.cpp: qualify two RouteHealth *h locals as const — only read for stale-route checks, never mutated through the pointer - Router.cpp: qualify meshtastic_NodeInfoLite *node as const in shouldDecrementHopLimit — only read for favorite/role predicate - test_position_module/test_main.cpp: change bare PositionModule.h to modules/PositionModule.h — build_flags sets -Isrc, not -Isrc/modules, so the bare form fails to resolve in the native PlatformIO test env Co-Authored-By: Claude Sonnet 4.6 * WarmStore: cache device role + protected category in last_heard low bits Steal the low 6 bits of WarmNodeEntry.last_heard to carry an evicted node's device role (4 bits) and a protected category (2 bits) for the hop-trim path, at zero record-size cost (entry stays 40 B; no RAM/flash growth). The high bits remain a real unix-seconds timestamp, quantised to 64 s — ample for warm LRU ordering of long-tail nodes. - absorb() packs role/protectedCat; place()/ring replay store the raw word so metadata round-trips through flash. LRU compares masked time (warmTimeOf). - take() rehydration masks the metadata bits and restores the cached role so a re-admitted node isn't stuck at CLIENT until its next NodeInfo. - NodeDB classifies the category (favorite/ignored/verified -> Flag; tracker/sensor/tak_tracker -> Role) at each eviction site. - WarmNodeStore::lookupMeta() exposes role/category to consumers. - Bump WARM_RING_MAGIC (WRNG->WRN2): old rings read as erased and rebuild; warm data is a non-critical evictee cache, so discard-on-upgrade is safe. Tests: test_warm_store 11/11 (new meta round-trip + quantisation-aware ordering); NodeDB compiles (test_nodedb_blocked 4/4). Co-Authored-By: Claude Opus 4.8 (1M context) * WarmStore: migrate v1 rings/files by discarding last_heard, not the data Previously the WRNG->WRN2 magic bump treated old rings as erased, discarding all warm entries — including the PKI public keys that let evicted nodes keep decrypting DMs. Instead, read v1 (WRNG / WRM1) records and keep each node's identity + public key, discarding only last_heard (its low bits would otherwise be misread as the new role/protected metadata). Records re-rank and re-learn their role on next contact. - Ring backend (nRF52840): ringReadHeader accepts both magics and reports v1 via an out-param; replay zeroes last_heard for v1 records. If the active head page is v1, force a rotation so new v2 records never land in a v1-headered page (which would discard their freshly-set role on the next load). Legacy pages convert to v2 as the ring rotates. - File backend (warm.dat): bump WARM_STORE_MAGIC WRM1->WRM2; accept WRM1, verify CRC against the stored bytes, then discard last_heard and mark dirty so the next save rewrites as v2. Tests: test_warm_store 12/12 (adds test_ws_v1_migration_discardsLastHeard: key survives, role/protected reset). Co-Authored-By: Claude Opus 4.8 (1M context) * WarmStore: guard role bit-width + test eviction carries role/protected - static_assert that the device role enum still fits the 4-bit warm metadata field (WARM_ROLE_MASK); fails the build loudly if a new role is added past 15 rather than silently truncating role on eviction. (Max role today = 12.) - Add test_migration_carriesRoleAndProtectedIntoWarm: a demoted TRACKER lands in the warm tier with its key, role=TRACKER and protected category=Role; a demoted CLIENT carries role=CLIENT/None. Exercises the NodeDB eviction path + warmProtectedCategory classification (the warm-store unit tests only cover absorb() directly). Tests: test_nodedb_blocked 5/5. Co-Authored-By: Claude Opus 4.8 (1M context) * fix copilot comments * fix(test): restore #if HAS_TRAFFIC_MANAGEMENT guard in TMM test The rebase onto PR1.5 lost the top-level HAS_TRAFFIC_MANAGEMENT guard that PR1.5 introduced, leaving the #else/#endif tail orphaned and causing compile errors on non-TMM builds. Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: Ben Meadors --- docs/nexthop-routing-reliability.md | 456 ++++++++++++ .../mesh/test_nexthop_multihop_recovery.py | 347 +++++++++ src/mesh/Default.h | 2 +- src/mesh/MeshTypes.h | 5 + src/mesh/NextHopRouter.cpp | 217 +++++- src/mesh/NextHopRouter.h | 57 ++ src/mesh/NodeDB.cpp | 126 +++- src/mesh/NodeDB.h | 31 + src/mesh/PacketHistory.cpp | 6 +- src/mesh/ReliableRouter.cpp | 4 + src/mesh/Router.cpp | 41 +- src/mesh/WarmNodeStore.cpp | 97 ++- src/mesh/WarmNodeStore.h | 52 +- src/mesh/mesh-pb-constants.h | 8 +- src/modules/TraceRouteModule.cpp | 12 + src/modules/TrafficManagementModule.cpp | 695 +++++++----------- src/modules/TrafficManagementModule.h | 258 ++----- test/test_nexthop_routing/test_main.cpp | 465 ++++++++++++ test/test_nodedb_blocked/test_main.cpp | 36 +- test/test_traffic_management/test_main.cpp | 259 +++++-- test/test_warm_store/test_main.cpp | 92 ++- 21 files changed, 2529 insertions(+), 737 deletions(-) create mode 100644 docs/nexthop-routing-reliability.md create mode 100644 mcp-server/tests/mesh/test_nexthop_multihop_recovery.py create mode 100644 test/test_nexthop_routing/test_main.cpp diff --git a/docs/nexthop-routing-reliability.md b/docs/nexthop-routing-reliability.md new file mode 100644 index 000000000..114bd5687 --- /dev/null +++ b/docs/nexthop-routing-reliability.md @@ -0,0 +1,456 @@ +# NextHop direct-message reliability on dense meshes — findings & plan + +**Status:** Implemented — mitigations and tests in `PR3-tmm-nexthop` +**Date:** 2026-06-13 +**Area:** `src/mesh` router stack (`NextHopRouter`, `ReliableRouter`, `FloodingRouter`, `Router`, `NodeDB`, `PacketHistory`) +**Constraint:** No over-the-air / wire-format changes — `next_hop` and `relay_node` stay 1 byte, no `PacketHeader` changes, no breaking protobuf changes. All new state is RAM-only. + +This document captures the analysis and the proposed mitigations so the work can be +continued on this branch by anyone. It is intentionally code-grounded (file:line +references throughout) and standalone — you should not need the original investigation +context to pick it up. + +--- + +## TL;DR + +NextHop routing for direct messages (DMs) is unreliable on dense meshes. The headline +cause is the **birthday problem**: `next_hop` and `relay_node` are each a single byte +(the last byte of a 32-bit node number), so on a mesh of N nodes the probability that +two share the same byte hits ~50% at **~19 nodes** and is near-certain by 50–100. But +there are **other, equally important issues**: that single byte is trusted blindly at +five different code sites, learned routes **never decay**, routes are learned from the +**reverse (ACK) path** (asymmetric-link hazard), and collision-driven spurious +rebroadcasts **amplify congestion** exactly when the mesh is busy. + +Because we can't widen the on-wire field, the fix is **interpretation-side** ("don't +trust a byte that doesn't map to a unique reachable neighbor — flood instead") plus +**recovery-side** ("decay stale/failing routes so they get re-discovered"). Four +mitigations, M1–M4, all RAM-only. The net behavioral change: on dense/mobile meshes a +DM that today silently misroutes or black-holes instead falls back to managed flooding +(which still delivers) and re-learns a fresh route quickly. Sparse-mesh happy paths are +unchanged. + +--- + +## How NextHop routing works today (mechanics) + +Inheritance chain: `Router` → `FloodingRouter` → `NextHopRouter` → `ReliableRouter`. + +**The single-byte identifiers.** Both routing bytes come from one helper: + +```cpp +// src/mesh/NodeDB.h:255 +uint8_t getLastByteOfNodeNum(NodeNum num) { return (uint8_t)((num & 0xFF) ? (num & 0xFF) : 0xFF); } +``` + +It projects a 32-bit node number onto 255 values (`0x00` is remapped to `0xFF` so it +never collides with the `0`-valued sentinels `NO_NEXT_HOP_PREFERENCE` / `NO_RELAY_NODE`, +`src/mesh/MeshTypes.h:44-46`). `next_hop` and `relay_node` in the packet header are +`uint8_t` (`src/mesh/mesh.pb.h`, comments "Last byte of the node number…"). The learned +route stored per destination, `meshtastic_NodeInfoLite::next_hop`, is also a single byte +(`src/mesh/generated/meshtastic/deviceonly.pb.h:83`). + +**Sending a DM** — `NextHopRouter::send` (`src/mesh/NextHopRouter.cpp:23`): + +1. `p->relay_node = getLastByteOfNodeNum(getNodeNum())` (mark ourselves as relayer). +2. `p->next_hop = getNextHop(p->to, p->relay_node)` (`src/mesh/NextHopRouter.cpp:192`): + look up `nodeDB->getMeshNode(to)->next_hop`; return it unless it equals the relayer + byte; otherwise `NO_NEXT_HOP_PREFERENCE` (→ flood). + +**Relaying** — `NextHopRouter::perhapsRebroadcast` (`src/mesh/NextHopRouter.cpp:133`): +rebroadcast iff `next_hop == NO_NEXT_HOP_PREFERENCE` (flood) **or** +`next_hop == getLastByteOfNodeNum(getNodeNum())` (we are the addressed next hop) +(`:147`). Each node only ever compares against **its own** byte. + +**Learning** — `NextHopRouter::sniffReceived` (`src/mesh/NextHopRouter.cpp:89`): on an +ACK/reply (`request_id`/`reply_id` set), if the relayer of the ACK was also a relayer of +the original packet (validated via `PacketHistory::checkRelayers`), set +`origTx->next_hop = p->relay_node` (`:114`). I.e. the **forward** next-hop is learned +from the **reverse** path's relayer. + +**Retransmission / fallback** — `NextHopRouter::doRetransmissions` +(`src/mesh/NextHopRouter.cpp:284`). Budgets: `NUM_RELIABLE_RETX=3` (originator: initial + +- 2 retries), `NUM_INTERMEDIATE_RETX=2` (relayer: 1 retry). On the **last** retry + (`numRetransmissions==1`) it resets `next_hop` to `NO_NEXT_HOP_PREFERENCE` on the packet + **and** clears `sentTo->next_hop` in NodeDB, then floods (`:313-321`). Retransmit timing + comes from `iface->getRetransmissionMsec`, whose contention window **grows with channel + utilization** (`src/mesh/RadioInterface.cpp` `getTxDelayMsec`/`getTxDelayMsecWeighted`). + +**Dedup / relayer history** — `PacketHistory` (`src/mesh/PacketHistory.cpp`): a bounded +ring (`PACKETHISTORY_MAX = max(MAX_NUM_NODES*2, 100)`, 20 B/record) keyed by +`(sender,id)`, tracking up to `NUM_RELAYERS=6` relayer **bytes** per packet in +`relayed_by[]`. `wasRelayer` (`:490`) and `checkRelayers` (`:517`) match bytes against +that array. + +--- + +## Root-cause analysis + +### 1. The single byte is trusted blindly at five sites (the birthday problem) + +| # | Site | File:line | Failure on collision | +| --- | -------------------------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| 1 | Rebroadcast self-check | `NextHopRouter.cpp:147` | A remote "impostor" node sharing the intended next-hop's byte also rebroadcasts → wasted airtime / congestion. | +| 2 | Route learning | `NextHopRouter.cpp:111-114` | Stores an ambiguous byte as the route; later resolves to the wrong physical node. | +| 3 | Relayer validation | `PacketHistory.cpp:490-538` | `wasRelayer(byte)` returns true for the wrong node → mis-validated ACK / mis-learn. | +| 4 | Favorite-router hop preservation | `Router.cpp:120-145` | **First** NodeDB node whose last byte matches wins — non-deterministic; can preserve hops for the wrong relay (hop leak). | +| 5 | Send-path lookup | `NextHopRouter.cpp:192-207` | Emits a byte that may address the wrong node; no check it still maps to a reachable neighbor. | + +Collision math (uniform last byte over 255 buckets): P(collision) ≈ 50% at ~19 nodes, + +> 99% by ~75 nodes. Dense meshes are squarely in the "always colliding" regime. + +### 2. Stale routes never decay + +The learned `next_hop` byte is cleared only on the **current DM's** last retry +(`NextHopRouter.cpp:313-321`). A route learned hours ago that has since gone dead is +still trusted on the **next** DM's first attempt — which on a congested mesh is also the +slowest attempt. Result: silent black-hole at a dead hop until the retransmission budget +drains, then a late flood. Intermediate nodes hold stale routes indefinitely. + +### 3. Reverse-path (asymmetric-link) learning + +`origTx->next_hop` is learned from the ACK's relayer (`NextHopRouter.cpp:110-114`) — the +**reverse** direction. RF links are frequently asymmetric, so the best reverse relay can +be a poor forward relay. Worse, the next reverse ACK immediately re-learns the same bad +hop, so the route **flaps** back to the bad value even after a failure reset. + +### 4. Congestion amplification + +Collision-driven impostor rebroadcasts (issue 1) add airtime; the contention window +grows with channel utilization, so retransmit intervals **lengthen** exactly when the +mesh is busy. The 3-try reliable budget can then expire before delivery. On dense +meshes, efficiency _is_ reliability. + +### Note: pubkey-derived node numbers (develop / 2.8) — does not change the plan + +develop derives the node number from the public key: +`my_node_num = crc32Buffer(public_key)` (`src/mesh/NodeDB.cpp:481`), re-derived on key +change in `createNewIdentity()` (`src/mesh/NodeDB.cpp:3113`). This **reinforces** the +plan rather than changing it: + +- **Birthday problem unchanged and now textbook-exact.** CRC32 mixes well → the last + byte is uniformly distributed over 256 values. Derivation adds no wire bits. +- **Node numbers are now immutable / identity-bound.** Pre-2.8 `pickNewNodeNum()` could + renumber a node to dodge a conflict; now the number is fixed by the key, so a last-byte + collision **cannot be resolved operationally by renumbering** → M1/M2/M3 become _more_ + necessary. +- **Resolver gets cleaner inputs.** Stable node numbers keep a learned byte bound to one + identity (good for M3 freshness). `createNewIdentity()` retires the old entry by marking + it **ignored** and clearing its pubkey (`src/mesh/NodeDB.cpp:3123-3125`), which M1's + candidate gate already skips — so key rotation can't pollute resolution. +- **No wire-free disambiguation unlocked.** A receiver still gets only 1 byte and cannot + recover which full node number a colliding value meant — so "detect ambiguity → flood" + remains the correct strategy. + +--- + +## Proposed mitigations + +Key insight for all of M1/M2: **a 1-byte ID only needs to be unique among a node's +direct neighbors / plausible relays, not the whole mesh.** That candidate set is small +(typically 5–15), so a byte usually resolves unambiguously there; when it doesn't, fall +back to the _safe_ behavior (flood / decrement / don't-learn). + +### M1 — Ambiguity-aware last-byte resolution (new NodeDB primitive) + +New types + methods in `src/mesh/NodeDB.h` (near line 255) / `src/mesh/NodeDB.cpp` +(near `getMeshNode`, ~2936): + +```cpp +enum class LastByteResolution : uint8_t { None, Unique, Ambiguous }; +struct ResolvedNode { LastByteResolution status = LastByteResolution::None; NodeNum num = 0; }; + +// Resolve a single on-wire last-byte to a unique full NodeNum among relevant candidates. +ResolvedNode resolveLastByte(uint8_t lastByte, bool requireDirectNeighbor); +// Convenience: true iff exactly one relevant candidate (Ambiguous and None both -> false = SAFE). +bool resolveUniqueLastByte(uint8_t lastByte, bool requireDirectNeighbor, NodeNum *outNum = nullptr); +``` + +- **One linear pass** over `meshNodes`, reusing `getNumMeshNodes()`/`getMeshNodeByIndex()`, + the bitfield helpers (`nodeInfoLiteIsFavorite/HasUser/IsIgnored`), `sinceLastSeen()`, + and `getLastByteOfNodeNum()`. **Early-exit** on the 2nd match (return `Ambiguous`). +- **Guard:** `if (lastByte == 0) return {None, 0};` (covers `NO_RELAY_NODE` / MQTT-invalid). +- **Candidate gate** (skip): `num == getNodeNum()` (never resolve to ourselves), `num == 0`, + `num == NODENUM_BROADCAST`, `nodeInfoLiteIsIgnored`. Then match + `getLastByteOfNodeNum(node->num) == lastByte` (cheapest test last, mirroring `Router.cpp:119`). +- **Relevance gate:** + - `requireDirectNeighbor == true` (strict, for SEND): `has_hops_away && hops_away == 0` + **and** `sinceLastSeen(node) < NEXTHOP_NEIGHBOR_FRESH_SECS`. + - `requireDirectNeighbor == false` (lenient, for learn / hop-preserve): accept if direct + neighbor **or** `nodeInfoLiteIsFavorite` **or** role ∈ {ROUTER, ROUTER_LATE, CLIENT_BASE}. +- **No tie-break.** A collision must return `Ambiguous` — picking "best SNR" would + resurrect the silent-misroute bug. (Deliberate non-goal; document in code.) + +New constant in `src/mesh/MeshTypes.h` (near line 44): +`#define NEXTHOP_NEIGHBOR_FRESH_SECS (60 * 60 * 2)` (mirrors `NUM_ONLINE_SECS`). + +### M2 — Only route on bytes that resolve to a unique, reachable neighbor + +In `getNextHop` (`src/mesh/NextHopRouter.cpp:192-207`), after the existing split-horizon +check (`node->next_hop != relay_node`), require the stored byte to resolve to a **unique, +currently-fresh direct neighbor**; else flood: + +```cpp +if (node->next_hop != relay_node) { + ResolvedNode r = nodeDB->resolveLastByte(node->next_hop, /*requireDirectNeighbor=*/true); + if (r.status == LastByteResolution::Unique) return node->next_hop; + LOG_WARN("Next hop 0x%x for 0x%x %s -> flood", node->next_hop, to, + r.status == LastByteResolution::Ambiguous ? "ambiguous among neighbors" : "no longer a neighbor"); + return std::nullopt; +} +``` + +This self-heals when a neighbor goes away (unicast-into-a-void becomes a flood). It +applies to originating, relaying, and retrying, since all route through `getNextHop`. + +Apply M1's safe fallback at the other sites: + +- **Learning** (`NextHopRouter.cpp:111-114`): gate `origTx->next_hop = p->relay_node` on + `resolveUniqueLastByte(p->relay_node, /*direct=*/false)`. Ambiguous/unknown → don't + learn (leave route unset → flood). +- **Favorite-router preservation** (`Router.cpp:120-145`): replace the "first match wins" + loop with `resolveUniqueLastByte(p->relay_node, /*direct=*/false)` + a re-check that the + resolved node is favorite/has_user/router. Ambiguous/none/not-favorite → **decrement** + (safe). Net: removes one full DB scan, adds one resolver scan (wash). + +**Left unchanged, by design (document why in code):** + +- **Site 1** rebroadcast self-check (`NextHopRouter.cpp:147`) and self-identity checks + (`ReliableRouter.cpp:127`): a node matches its **own** byte — no DB resolution helps. A + remote impostor sharing the intended next-hop's byte will still rebroadcast. M1/M2 + shrink the blast radius by reducing how often an ambiguous byte is ever stored or + originated; a true fix needs a wider field (out of scope). **This is the one residual + the plan cannot fully close.** +- **Site 3** `wasRelayer`/`checkRelayers` (`PacketHistory.cpp:490-538`): intentionally + byte-domain (both sides are on-wire bytes); the consumer (learning) is now hardened. + Add a one-line comment; do not change. + +### M3 — Route freshness / failure memory (RAM table on NextHopRouter) + +A bounded, LRU-evicted table keyed by destination, mirroring `PacketHistory`'s +reuse-oldest discipline (not an unbounded map) to cap RAM. + +`src/mesh/NextHopRouter.h` (near `pending`, line 99): + +```cpp +struct RouteHealth { + NodeNum dest = 0; // 0 == empty slot + uint32_t learnedAtMsec = 0; // millis() at last (re)learn; rollover-aware + uint8_t consecutiveFailures = 0; + uint8_t lastNextHop = NO_NEXT_HOP_PREFERENCE; // byte this health refers to +}; +static constexpr uint8_t ROUTE_HEALTH_MAX = 32; // ~384B; drop to 16 if RAM-tight +RouteHealth routeHealth[ROUTE_HEALTH_MAX] = {}; +// Helpers take `now` (pure/testable): findRouteHealth, getOrAllocRouteHealth, +// noteRouteLearned, noteRouteSuccess, noteRouteFailure, isRouteStale, clearRouteHealth +``` + +Policy: + +| Constant | Value | Rationale | +| ------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `ROUTE_TTL_MSEC` | 30 min | Survives a normal conversation; re-discovers a moved node within a telemetry interval. | +| `ROUTE_FAILURE_THRESHOLD` | 3 | 1–2 consecutive failures are transient LoRa collisions; 3 to the same hop = dead. Accumulates **across** DMs (independent of the per-DM 3-try budget). | + +`isRouteStale(h, now)` = `(now - h.learnedAtMsec) >= ROUTE_TTL_MSEC || h.consecutiveFailures >= ROUTE_FAILURE_THRESHOLD`. +All age math uses **unsigned subtraction** (rollover-safe, matching +`PacketHistory.cpp:364`); treat `learnedAtMsec == 0` as "set now". + +Wiring (as built — `src/mesh/NextHopRouter.cpp`, `src/mesh/ReliableRouter.cpp`): + +- `getNextHop`: if a health record matches the stored byte and `isRouteStale`, clear + `node->next_hop` (NodeDB) **and** `clearRouteHealth`, return `nullopt` (flood). No + record yet (cold path, first DM after boot) → trust NodeDB, but the M2 strict-neighbor + gate still applies. +- `sniffReceived` learn: gate the write through `resolveUniqueLastByte` (M2), then + `noteRouteLearned(p->from, p->relay_node, millis())` — resets `consecutiveFailures` + **only if the hop changed** (anti-flap for asymmetric re-learn); otherwise just refreshes + `learnedAtMsec`. (No success signal is taken on the intermediate reverse-pass: an ACK + merely passing through us is not proof that _we_ delivered, and resetting failures there + would reintroduce the asymmetric flap.) +- `doRetransmissions`: on the last-retransmission branch (`numRetransmissions == 1`, the + point a directed delivery has gone un-ACKed for both originator and intermediate) → + `noteRouteFailure(to)`, then the existing NodeDB `next_hop` reset + flood. We deliberately + do **not** `clearRouteHealth` here: keeping the record is what lets the failure count + accumulate across DMs so a flapping reverse-path-relearned dead hop eventually ages out. +- `ReliableRouter::sniffReceived` ACK path → `noteRouteSuccess(getFrom(p), millis())` + (an end-to-end ACK addressed to us is genuine forward-delivery proof; clears failures and + refreshes freshness). `noteRouteSuccess`/`noteRouteFailure` are no-ops when no record + exists, so flood-only destinations never pollute the table. + +**Reconciliation (no double-handling):** `doRetransmissions` owns _in-flight_ failure of +the current DM (reset NodeDB `next_hop` + flood, and bump the cross-DM failure counter); +`getNextHop` owns _between-DM_ staleness (TTL or failure-threshold → flood + clear). The +only place that erases a health record is the `getNextHop` decay path; the retransmission +path leaves it intact so the counter survives a reverse-path re-learn. + +### M4 — Earlier flood for unverified routes (gated, off by default) + +Compile-gated so healthy sparse meshes are untouched. **Default is off** — the define +lives in `NextHopRouter.h` and must be flipped to measure: +`#define NEXTHOP_EARLY_FLOOD_ON_UNVERIFIED 1`. + +In `doRetransmissions`, the directed-retry `else` branch: if the route is **not verified** +(`!findRouteHealth(to) || consecutiveFailures > 0 || isRouteStale`), reset `next_hop` and +flood on this attempt instead of spending another directed try. A **verified** route +(record present, `consecutiveFailures == 0`, within TTL — i.e. recently ACKed) takes the +unchanged directed-retry path, so the sparse-mesh happy path is untouched. Trade-off: +airtime ↔ latency; the gate ensures we never pay the flood cost on a proven route, only on +one we already distrust. Off by default precisely so it can be A/B-measured on the +simulator before broad enable. + +--- + +## Files to modify + +| File | Change | +| ------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| `src/mesh/MeshTypes.h` | `NEXTHOP_NEIGHBOR_FRESH_SECS`, `ROUTE_TTL_MSEC`, `ROUTE_FAILURE_THRESHOLD`, `NEXTHOP_EARLY_FLOOD_ON_UNVERIFIED` | +| `src/mesh/NodeDB.h` / `src/mesh/NodeDB.cpp` | `LastByteResolution`, `ResolvedNode`, `resolveLastByte`, `resolveUniqueLastByte` | +| `src/mesh/NextHopRouter.h` | `RouteHealth` + array + helpers; `#ifdef PIO_UNIT_TESTING public:` for helpers and `getNextHop` | +| `src/mesh/NextHopRouter.cpp` | `getNextHop` (M2 gate + M3 decay); `sniffReceived` (learn gate + health seed + success); `doRetransmissions` (failure counting + M4); comment site 1 | +| `src/mesh/Router.cpp` | `shouldDecrementHopLimit` → resolver + favorite/router re-check | +| `src/mesh/ReliableRouter.cpp` | ACK path → `noteRouteSuccess` | +| `test/test_nexthop_routing/test_main.cpp` | **new** unit suite (auto-built under `[env:native]`) | + +**Reuse, don't reinvent:** `getLastByteOfNodeNum`, `sinceLastSeen`, the bitfield helpers, +`getMeshNodeByIndex`/`getNumMeshNodes`, PacketHistory's reuse-oldest eviction shape, and +`MockNodeDB::addTestNode` (from `test/test_hop_scaling/test_main.cpp`). + +--- + +## Edge cases + +- **`0x00`↔`0xFF` projection:** the resolver compares via `getLastByteOfNodeNum` on both + sides, so a `…00` node and a `…FF` node correctly collide on `0xFF` → `Ambiguous`. Test + explicitly. +- **MQTT packets:** `relay_node`/`next_hop` are forced invalid when `hop_start == 0` + (`src/mesh/RadioLibInterface.cpp:603-605`) → byte 0 → resolver `None` → don't learn + (correct). +- **`has_hops_away == false`** nodes are excluded from the strict gate (never fabricate a + Unique neighbor for M2); admitted to the lenient gate only via favorite/router role. + Safe; self-corrects once `hops_away` is learned. +- **Self / broadcast:** the resolver skips `getNodeNum()` and `NODENUM_BROADCAST`; + `getNextHop` already early-returns for broadcast. +- **Perf:** M2 adds one O(N) resolver scan per directed send/relay (early-exit on the 2nd + match), cheaper than the crypto already on that path; site-4 is a wash. If ever hot, a + future 256-entry last-byte index is the optimization (not now — RAM). + +--- + +## Verification (all tiers) + +### 1. Native unit tests — new `test/test_nexthop_routing/test_main.cpp` + +`pio test -e native -f test_nexthop_routing`; on macOS `./bin/test-native-docker.sh -f test_nexthop_routing`. +Design the RouteHealth helpers to take `now` as a parameter so the 30-min TTL logic is +testable without a clock mock. + +- **Resolver:** None / Unique / **Ambiguous (birthday collision)** / strict-excludes-stale / + strict-excludes-far / lenient-includes-favorite-router / lenient-collision / skips-self / + skips-ignored / **`0x00`↔`0xFF` collision** / early-exit. +- **`getNextHop`:** unique→byte, **ambiguous→nullopt**, stale-neighbor→nullopt, + split-horizon (relay==next_hop)→nullopt, broadcast→nullopt. +- **RouteHealth:** TTL boundary, **rollover** (learn near `0xFFFFFFFF`, check after wrap), + failure threshold, success-resets, **re-learn-same-hop keeps fails (anti-flap)**, + re-learn-new-hop resets, LRU eviction bound, clear. +- **Site-4:** preserve on unique favorite router; **decrement on two colliding favorites**; + decrement when the resolved node is not a favorite. +- **Sparse-mesh regression:** all-distinct last bytes → every resolve Unique, `getNextHop` + returns the stored byte unchanged (proves no happy-path change). +- Re-run `test_packet_history` and `test_hop_scaling` for no regression. + +### 2. portduino SimRadio simulator + +`pio run -e native && ./bin/test-simulator.sh`. Best vehicle for the **intermediate-node** +path the 2-device bench can't reach. Line topology A — B — C: establish A→C (B learns a +directed route), stop B relaying that dest, confirm A re-discovers via flood within +`ROUTE_FAILURE_THRESHOLD` and that B's `noteRouteFailure`/`clearRouteHealth` fires (visible +via the `LOG_INFO "Route to … stale"` / "Resetting next hop" lines). Use this to A/B M4 +(attempts-to-delivery, total airtime). + +### 3. Hardware via meshtastic MCP (auto-detect; 3+ devices for a real hop) + +- `mcp-server/tests/mesh/test_nexthop_multihop_recovery.py` — **the multi-hop validator + for this work** (added on this branch). Self-discovers an A — relay — C line, asserts a + directed DM is delivered across the relay (next_hop + M1/M2/M3 engaged), and asserts + delivery recovers after the relay is power-cycled (M3). Skips unless the bench is a true + multi-hop line (≥3 roles via `--hub-profile`, endpoints out of direct RF range). +- `mcp-server/tests/mesh/test_direct_with_ack.py` — happy-path regression: a fresh/unique + route still delivers a want_ack DM on the first/second try (M4's gate must keep this + green). +- `mcp-server/tests/mesh/test_peer_offline_recovery.py` — 2-device recovery validator: peer + off mid-conversation then back. Must stay green and ideally recover in fewer attempts. + +### 4. Build / format sanity + +native-macos **and** Docker both ways; trunk clang-format@16.0.3; a release `pio run` to +confirm the `#ifdef PIO_UNIT_TESTING` visibility widening does **not** leak into +production; sanity-check RAM headroom on the smallest nRF52 build for the ~384 B table. + +--- + +## Verification status (as built on `nexthop-redux`) + +| Tier | What ran | Result | +| -------------------------------- | ----------------------------------------------------------------------------------- | ------------------- | +| Unit (native-macos) | `test_nexthop_routing` (31 cases) | ✅ 31/31 | +| Unit (Docker / Linux, CI parity) | `test_nexthop_routing` | ✅ 31/31 | +| Regression | `test_packet_history`, `test_hop_scaling`, `test_mqtt`, `test_traffic_management` | ✅ 105/105 | +| Build | `pio run -e native-macos` (M4 off) and with `-DNEXTHOP_EARLY_FLOOD_ON_UNVERIFIED=1` | ✅ both link | +| Format | trunk `clang-format@16.0.3` | ✅ no issues | +| Simulator (CI `simulator-tests`) | `meshtasticd -s` + `meshtastic.test.testSimulator()` on native-macos | ✅ exit 0, no crash | + +**Pending (environment-blocked, not yet run):** + +- **Multi-hop A–B–C recovery sim** — the `simulator/` broker hub is **not git-tracked** + (only stale local `.pyc`), and two `meshtasticd -s` instances can't hear each other + without it. The intermediate-node failure-count path and the M4 A/B therefore have unit + coverage of their logic but no end-to-end multi-node run yet. +- **Hardware / multi-hop tier** — a committable bench test now exists: + `mcp-server/tests/mesh/test_nexthop_multihop_recovery.py`. It self-discovers a real + multi-hop pair (A — relay — C), asserts a directed DM is delivered across the relay, and + asserts delivery recovers after the relay is power-cycled (the M3 path). It + `pytest.skip`s cleanly unless the bench is a true line with endpoints out of direct RF + range (≥3 roles via `--hub-profile`), so it's safe to commit and only asserts when the + NextHop path is genuinely exercised. Collected + verified to skip without hardware; + not yet run on a bench. `test_direct_with_ack.py` / `test_peer_offline_recovery.py` + remain the 2-device happy-path/recovery regressions. + +--- + +## Risks & limitations + +- **Site-1 impostor rebroadcast** is unfixable without a wider field — documented; M1/M2 + only shrink its frequency. +- **Dense meshes flood DMs more often** — intended (a flooded DM arrives; a mis-unicast one + black-holes). Call out in the PR so reviewers expect a slightly higher DM flood rate on + very dense meshes. +- **M4 airtime** if the gate is too loose → default conservative + compile-gated + + simulator A/B before broad enable. +- **RAM** ~384 B (32 slots); 16 slots (~192 B) with graceful LRU degradation if tight. +- **Asymmetric flap** not fully closed (a _new_ bad hop resets the counter); the TTL + backstop bounds it. Per-hop failure history is future work (more RAM). + +--- + +## How to continue this work (commit sequencing) + +Each step is independently testable; land them as separate commits. + +1. **M1 resolver + unit tests** — `NodeDB` only; no behavior change until wired. Lands the + `resolveLastByte`/`resolveUniqueLastByte` primitive and its full unit-test matrix. +2. **M2 + wiring + tests** — `getNextHop` strict gate, learning gate, favorite-router + preservation rewrite. Adds the `getNextHop` and site-4 tests. +3. **M3 health table + decay + tests** — RAM `RouteHealth` table, decay-on-read, failure/ + success accounting, reconciliation with the existing last-retry reset. Adds the + route-health unit tests and the simulator recovery check. +4. **M4 gated tuning** — early-flood-on-unverified behind the compile flag; simulator A/B + and hardware regression. + +Reference plan (with the same content) was developed at +`~/.claude/plans/nexthop-routing-for-direct-lexical-shell.md` on the author's machine; this +in-repo doc is the canonical handoff copy. diff --git a/mcp-server/tests/mesh/test_nexthop_multihop_recovery.py b/mcp-server/tests/mesh/test_nexthop_multihop_recovery.py new file mode 100644 index 000000000..a61645c91 --- /dev/null +++ b/mcp-server/tests/mesh/test_nexthop_multihop_recovery.py @@ -0,0 +1,347 @@ +"""Multi-hop NextHop directed-message delivery + relay-recovery (bench test). + +This is the hardware/tier-3 validator for the NextHop DM reliability work +(see `docs/nexthop-routing-reliability.md`). The unit suite +`test/test_nexthop_routing` covers the routing *logic* exhaustively; this test +covers the *end-to-end* multi-hop behavior that only a real (or RF-separated) +mesh exercises: + + * a directed DM that must traverse a relay is delivered (next_hop routing + + the M1/M2 ambiguity gate + M3 route learning all engage), and + * when the established relay drops and returns, delivery recovers rather than + black-holing (the M3 stale-route decay / re-learn path). + +TOPOLOGY REQUIREMENT — why this usually SKIPS: + A NextHop relay only happens when the two endpoints are NOT direct neighbors. + Three co-located radios all hear each other, so A→C is a single direct hop and + next_hop never engages. To run this test the bench must be a *line* — A — B — C + — with the endpoints out of each other's direct RF range (physical distance or + attenuators). The `multihop_topology` fixture detects this automatically: it + warms the mesh, looks for a pair that is ≥1 hop apart, confirms the relay via + traceroute, and `pytest.skip`s cleanly when the bench is all-direct. So this + file is safe to commit and run anywhere — it only *asserts* when the topology + genuinely requires a relay. + +REQUIREMENTS: + * ≥3 baked devices. The default hub profile is 2 roles (nrf52, esp32s3); add a + third via `--hub-profile=path/to/hub.yaml` (see conftest `hub_profile`). + * The relay-recovery test additionally needs uhubctl + a power-controllable + relay port (same gate the other power tests use). +""" + +from __future__ import annotations + +import time +from typing import Any + +import pytest +from meshtastic_mcp.connection import connect +from tests import _power +from tests._port_discovery import resolve_port_by_role + +from ._receive import ReceiveCollector, nudge_nodeinfo, nudge_nodeinfo_port + + +def _hops_away(rec: dict[str, Any]) -> int | None: + """Read a node's hop distance from a `nodesByNum` entry, tolerating either + the camelCase (`hopsAway`) or snake_case (`hops_away`) spelling depending on + the meshtastic-python version.""" + for key in ("hopsAway", "hops_away"): + val = rec.get(key) + if isinstance(val, int): + return val + return None + + +def _warm_mesh(ports: list[str], rounds: int = 2, settle: float = 6.0) -> None: + """Flood a fresh NodeInfo from every node so the whole mesh (including + multi-hop pairs, reached via relayed broadcasts) populates pubkeys and hop + distances. Best-effort — a single node failing to nudge shouldn't abort.""" + for _ in range(rounds): + for port in ports: + try: + nudge_nodeinfo_port(port) + except Exception: # noqa: BLE001 — warmup is best-effort + pass + time.sleep(0.5) + time.sleep(settle) + + +def _wait_for_pubkey( + tx_iface: Any, rx_num: int, rx_port: str, deadline_s: float = 90.0 +) -> bool: + """Block until `tx_iface` holds `rx_num`'s public key (directed PKI sends + NAK without it). Re-nudges both sides periodically; multi-hop warmup is + slower than the 2-device case because NodeInfo must be relayed, hence the + longer default deadline.""" + deadline = time.monotonic() + deadline_s + last_nudge = time.monotonic() + while time.monotonic() < deadline: + rec = (tx_iface.nodesByNum or {}).get(rx_num, {}) + if rec.get("user", {}).get("publicKey"): + return True + if time.monotonic() - last_nudge > 20.0: + nudge_nodeinfo_port(rx_port) + nudge_nodeinfo(tx_iface) + last_nudge = time.monotonic() + time.sleep(1.0) + return False + + +def _traceroute_route(tx_port: str, rx_num: int, rx_port: str) -> list[int] | None: + """Run a traceroute TX→RX and return the forward `route` (list of relay node + numbers), or None if it couldn't be obtained. Mirrors test_traceroute's + request/PKI/retry pattern.""" + from meshtastic.mesh_interface import MeshInterface + + with ReceiveCollector(tx_port, topic="meshtastic.receive.traceroute") as tx: + nudge_nodeinfo_port(rx_port) + tx.broadcast_nodeinfo_ping() + if not _wait_for_pubkey(tx._iface, rx_num, rx_port, 60.0): + return None + for _attempt in range(2): + try: + tx._iface.sendTraceRoute(dest=rx_num, hopLimit=5) + break + except MeshInterface.MeshInterfaceError: + time.sleep(5.0) + else: + return None + pkt = tx.wait_for(lambda p: p.get("from") == rx_num, timeout=8.0) + if pkt is None: + return None + tr = (pkt.get("decoded", {}) or {}).get("traceroute") or {} + return [int(n) for n in (tr.get("route") or [])] + + +@pytest.fixture(scope="session") +def multihop_topology(baked_mesh: dict[str, Any]) -> dict[str, Any]: + """Discover a real multi-hop pier (tx → relay → rx) on the bench, or skip. + + Returns {tx_role, tx_port, rx_role, rx_port, rx_num, relay_role, relay_num}. + """ + roles = sorted(baked_mesh) + if len(roles) < 3: + pytest.skip( + "multi-hop NextHop test needs ≥3 baked devices arranged as a line " + "(endpoints out of direct RF range). Add a third role via " + f"--hub-profile. Detected roles: {roles}" + ) + + by_role = {r: (baked_mesh[r]["port"], baked_mesh[r]["my_node_num"]) for r in roles} + if any(num is None for _, num in by_role.values()): + pytest.skip("a baked device is missing my_node_num; can't map the topology") + + _warm_mesh([port for port, _ in by_role.values()]) + + # Find an ordered pair that is ≥1 hop apart, using each node's own nodeDB + # (cheap — no traceroute yet). On an all-direct bench nothing qualifies. + multihop_pair: tuple[str, str] | None = None + for a_role in roles: + a_port, _ = by_role[a_role] + try: + with connect(port=a_port) as a_iface: + nodes = a_iface.nodesByNum or {} + except Exception: # noqa: BLE001 + continue + for c_role in roles: + if c_role == a_role: + continue + _, c_num = by_role[c_role] + hops = _hops_away(nodes.get(c_num, {})) + if hops is not None and hops >= 1: + multihop_pair = (a_role, c_role) + break + if multihop_pair: + break + + if not multihop_pair: + pytest.skip( + "no multi-hop pair found — every device appears to be a direct " + "neighbor. Arrange the bench as a line (A — B — C) with the " + "endpoints out of direct RF range (distance or attenuators) so a " + "relay is actually required, then re-run." + ) + + a_role, c_role = multihop_pair + a_port, _ = by_role[a_role] + c_port, c_num = by_role[c_role] + + route = _traceroute_route(a_port, c_num, c_port) + if not route: + pytest.skip( + f"{a_role}→{c_role} looked multi-hop but traceroute returned no " + "intermediate relay; can't identify the relay node to drive the " + "recovery test" + ) + + relay_num = route[0] + relay_role = next((r for r in roles if by_role[r][1] == relay_num), None) + return { + "tx_role": a_role, + "tx_port": a_port, + "rx_role": c_role, + "rx_port": c_port, + "rx_num": c_num, + "relay_role": relay_role, + "relay_num": relay_num, + } + + +@pytest.mark.timeout(300) +def test_multihop_dm_delivers(multihop_topology: dict[str, Any]) -> None: + """A directed wantAck DM that must traverse the relay is delivered. + + Exercises the NextHop routing path end-to-end: TX picks a next hop toward + RX (M2 gate), the relay resolves the next_hop byte and forwards (M1), and + the route is learned from the returning ACK (M3). Retries absorb transient + LoRa loss; the assertion is on eventual delivery. + """ + tx_port = multihop_topology["tx_port"] + rx_port = multihop_topology["rx_port"] + rx_num = multihop_topology["rx_num"] + tx_role = multihop_topology["tx_role"] + rx_role = multihop_topology["rx_role"] + relay_role = multihop_topology["relay_role"] + + unique = f"nexthop-mh-{tx_role}-to-{rx_role}-{int(time.time())}" + + with ReceiveCollector(rx_port, topic="meshtastic.receive.text") as rx: + rx.broadcast_nodeinfo_ping() + with connect(port=tx_port) as tx_iface: + nudge_nodeinfo(tx_iface) + if not _wait_for_pubkey(tx_iface, rx_num, rx_port, 90.0): + pytest.skip( + f"{tx_role} never learned {rx_role}'s pubkey over the relay; " + "multi-hop PKI warmup didn't complete" + ) + got = None + for _attempt in range(3): + pkt = tx_iface.sendText(unique, destinationId=rx_num, wantAck=True) + assert pkt is not None + got = rx.wait_for( + lambda p: p.get("decoded", {}).get("text") == unique, + timeout=45, + ) + if got is not None: + break + rx.broadcast_nodeinfo_ping() + nudge_nodeinfo(tx_iface) + time.sleep(5.0) + + assert got is not None, ( + f"multi-hop directed DM {tx_role}→{rx_role} via relay " + f"{relay_role!r} never landed — NextHop multi-hop delivery is broken" + ) + + +@pytest.mark.timeout(600) +def test_multihop_relay_recovery( + multihop_topology: dict[str, Any], + power_cycle, # noqa: ARG001 — forces the uhubctl-availability skip +) -> None: + """Delivery recovers after the established relay drops and returns. + + Establishes a baseline DM (route via relay learned), powers the relay OFF + (confirming TX survives sending across a downed relay), then powers it back + ON and asserts directed delivery resumes — the M3 stale-route decay / + re-learn path. With a strict A — B — C line there is no path while B is down, + so we only assert TX doesn't crash during the outage; the delivery assertion + is after B returns. + """ + relay_role = multihop_topology["relay_role"] + if not relay_role: + pytest.skip( + "relay node isn't one of the baked hub roles, so it can't be " + "power-cycled; recovery test needs a controllable relay" + ) + + tx_port = multihop_topology["tx_port"] + rx_port = multihop_topology["rx_port"] + rx_num = multihop_topology["rx_num"] + tx_role = multihop_topology["tx_role"] + rx_role = multihop_topology["rx_role"] + + base = f"mh-recover-base-{int(time.time())}" + post = f"mh-recover-post-{int(time.time())}" + + # Baseline: confirm delivery works (so the route via the relay is learned) + # before we perturb anything — otherwise a later failure is ambiguous. + with ReceiveCollector(rx_port, topic="meshtastic.receive.text") as rx: + rx.broadcast_nodeinfo_ping() + with connect(port=tx_port) as tx_iface: + nudge_nodeinfo(tx_iface) + if not _wait_for_pubkey(tx_iface, rx_num, rx_port, 90.0): + pytest.skip("multi-hop PKI warmup failed; can't run recovery test") + tx_iface.sendText(base, destinationId=rx_num, wantAck=True) + assert ( + rx.wait_for( + lambda p: p.get("decoded", {}).get("text") == base, timeout=45 + ) + is not None + ), "baseline multi-hop delivery failed — skipping recovery to avoid a false result" + + # Power the relay OFF. + try: + _power.power_off(relay_role) + _power.wait_for_absence(relay_role, timeout_s=15.0) + except Exception as exc: # noqa: BLE001 + try: + _power.power_on(relay_role) + resolve_port_by_role(relay_role, timeout_s=30.0) + except Exception: # noqa: BLE001 + pass + pytest.skip(f"can't power-control relay {relay_role!r}: {exc}") + + # With the only relay down there's no path; we just confirm TX accepts the + # send and survives its internal retries (it must not crash / wedge). + try: + with connect(port=tx_port) as tx_iface: + pkt = tx_iface.sendText( + f"mh-while-down-{int(time.time())}", + destinationId=rx_num, + wantAck=True, + ) + assert pkt is not None + time.sleep(8.0) # let retransmissions + route decay run + except Exception as exc: # noqa: BLE001 — restore bench state before failing + _power.power_on(relay_role) + resolve_port_by_role(relay_role, timeout_s=30.0) + raise AssertionError( + f"TX crashed sending across a downed relay: {exc}" + ) from exc + + # Power the relay back ON and let it re-enumerate + boot. + _power.power_on(relay_role) + time.sleep(0.5) + try: + resolve_port_by_role(relay_role, timeout_s=30.0) + except Exception: # noqa: BLE001 — relay port isn't one we connect to directly + pass + time.sleep(8.0) + _warm_mesh([tx_port, rx_port], rounds=1) # re-flood so the relay re-learns + + # Delivery should resume once the relay is back (M3 re-learn / decay path). + got = None + with ReceiveCollector(rx_port, topic="meshtastic.receive.text") as rx: + rx.broadcast_nodeinfo_ping() + with connect(port=tx_port) as tx_iface: + nudge_nodeinfo(tx_iface) + _wait_for_pubkey(tx_iface, rx_num, rx_port, 90.0) + for _attempt in range(4): + pkt = tx_iface.sendText(post, destinationId=rx_num, wantAck=True) + assert pkt is not None + got = rx.wait_for( + lambda p: p.get("decoded", {}).get("text") == post, + timeout=45, + ) + if got is not None: + break + rx.broadcast_nodeinfo_ping() + nudge_nodeinfo(tx_iface) + time.sleep(6.0) + + assert got is not None, ( + f"after relay {relay_role!r} returned, multi-hop DM {tx_role}→{rx_role} " + "never resumed — stale-route recovery (M3) may be broken" + ) diff --git a/src/mesh/Default.h b/src/mesh/Default.h index 97f87fc7b..d638ca2a9 100644 --- a/src/mesh/Default.h +++ b/src/mesh/Default.h @@ -34,7 +34,7 @@ enum class TrafficType { POSITION, TELEMETRY }; // Traffic management defaults -#define default_traffic_mgmt_position_precision_bits 24 // ~10m grid cells +#define default_traffic_mgmt_position_precision_bits 19 // ~90m grid cells (±45m) #define default_traffic_mgmt_position_min_interval_secs (ONE_DAY / 2) // 12 hours between identical positions // Hop scaling defaults diff --git a/src/mesh/MeshTypes.h b/src/mesh/MeshTypes.h index 680926d3c..58ef32150 100644 --- a/src/mesh/MeshTypes.h +++ b/src/mesh/MeshTypes.h @@ -45,6 +45,11 @@ enum RxSource { // For old firmware there is no relay node set #define NO_RELAY_NODE 0 +// How recently we must have heard a direct neighbor for its single-byte relay id to be trusted as a +// unique next hop. Mirrors NUM_ONLINE_SECS (NodeDB.cpp). Used by NodeDB::resolveLastByte() to scope +// last-byte collision resolution to currently-reachable neighbors. +#define NEXTHOP_NEIGHBOR_FRESH_SECS (60 * 60 * 2) // 2 hrs + typedef int ErrorCode; /// Alloc and free packets to our global, ISR safe pool diff --git a/src/mesh/NextHopRouter.cpp b/src/mesh/NextHopRouter.cpp index e8613d457..5fe079a19 100644 --- a/src/mesh/NextHopRouter.cpp +++ b/src/mesh/NextHopRouter.cpp @@ -98,21 +98,38 @@ void NextHopRouter::sniffReceived(const meshtastic_MeshPacket *p, const meshtast // destination if (p->from != 0) { meshtastic_NodeInfoLite *origTx = nodeDB->getMeshNode(p->from); - if (origTx) { - // Either relayer of ACK was also a relayer of the packet, or we were the *only* relayer and the ACK came - // directly from the destination - // Single lookup for both relayer checks on the same (request_id, to) pair - bool wasAlreadyRelayer = false; - bool weWereSoleRelayer = false; - bool weWereRelayer = false; - checkRelayers(p->relay_node, ourRelayID, p->decoded.request_id, p->to, &wasAlreadyRelayer, &weWereRelayer, - &weWereSoleRelayer); - if ((weWereRelayer && wasAlreadyRelayer) || (getHopsAway(*p) == 0 && weWereSoleRelayer)) { - if (origTx->next_hop != p->relay_node) { // Not already set + // Either relayer of ACK was also a relayer of the packet, or we were the *only* relayer and the ACK came + // directly from the destination. checkRelayers is read-only on PacketHistory and O(1), so we run it even + // when origTx is absent — that lets us still capture the confirmed hop into the TMM overflow cache below. + // Single lookup for both relayer checks on the same (request_id, to) pair + bool wasAlreadyRelayer = false; + bool weWereSoleRelayer = false; + bool weWereRelayer = false; + checkRelayers(p->relay_node, ourRelayID, p->decoded.request_id, p->to, &wasAlreadyRelayer, &weWereRelayer, + &weWereSoleRelayer); + if ((weWereRelayer && wasAlreadyRelayer) || (getHopsAway(*p) == 0 && weWereSoleRelayer)) { + // M1/M2: only learn a next hop whose last byte maps to a single plausible relay. On a dense + // mesh the byte may be ambiguous; storing it would aim future DMs at the wrong node. This gate + // now protects BOTH the hot-store route (NodeInfoLite.next_hop) AND the TMM overflow cache — + // the overflow cache deliberately holds many more next-hop bytes (long-tail nodes), so it is + // even more collision-prone and must never store an ambiguous byte either. Ambiguous/unknown + // -> store nothing and keep flooding (safe). + if (nodeDB->resolveUniqueLastByte(p->relay_node, /*requireDirectNeighbor=*/false)) { + if (origTx && origTx->next_hop != p->relay_node) { // Not already set LOG_INFO("Update next hop of 0x%x to 0x%x based on ACK/reply (was relayer %d we were sole %d)", p->from, p->relay_node, wasAlreadyRelayer, weWereSoleRelayer); origTx->next_hop = p->relay_node; } + noteRouteLearned(p->from, p->relay_node, millis()); // M3: anchor freshness (hot or overflow route) +#if HAS_TRAFFIC_MANAGEMENT + // Mirror the confirmed (and now unique-resolved) hop into the TMM overflow cache so it + // survives even when the source isn't (or is no longer) in the hot NodeDB. + if (trafficManagementModule) + trafficManagementModule->setNextHop(p->from, p->relay_node); +#endif + } else { + LOG_DEBUG("Not learning next hop for 0x%x: relay byte 0x%x ambiguous/unknown; keep flooding", p->from, + p->relay_node); } } } @@ -144,6 +161,11 @@ bool NextHopRouter::perhapsRebroadcast(const meshtastic_MeshPacket *p) if (!isToUs(p) && !isFromUs(p) && (p->hop_limit > 0 || exhaustHops)) { if (p->id != 0) { if (isRebroadcaster()) { + // NOTE: this is a self-identity match (is the addressed next_hop OUR last byte?), so it + // cannot be hardened with resolveLastByte() — a remote node that legitimately shares our + // last byte will also match here and rebroadcast. That residual collision needs a wider + // on-wire field to fix. M1/M2 instead shrink the blast radius by reducing how often an + // ambiguous next_hop byte is ever learned (sniffReceived) or originated (getNextHop). if (p->next_hop == NO_NEXT_HOP_PREFERENCE || p->next_hop == nodeDB->getLastByteOfNodeNum(getNodeNum())) { meshtastic_MeshPacket *tosend = packetPool.allocCopy(*p); // keep a copy because we will be sending it LOG_INFO("Rebroadcast received message coming from %x", p->relay_node); @@ -194,15 +216,63 @@ std::optional NextHopRouter::getNextHop(NodeNum to, uint8_t relay_node) if (isBroadcast(to)) return std::nullopt; + // Hot store first: a direct array hit on the live NodeDB entry. meshtastic_NodeInfoLite *node = nodeDB->getMeshNode(to); if (node && node->next_hop) { + // M3: proactively decay a stale or repeatedly-failing route back to flooding, so a dead hop + // isn't trusted on the next DM's first (and on dense meshes, slowest) attempt. We only act on + // a health record that still matches the stored byte; a next_hop set by another path (e.g. + // TraceRouteModule) with no matching record is left authoritative. + const RouteHealth *h = findRouteHealth(to); + if (h && h->lastNextHop == node->next_hop && isRouteStale(*h, millis())) { + LOG_INFO("Next hop 0x%x for 0x%x is stale (age/fails); flood and clear", node->next_hop, to); + node->next_hop = NO_NEXT_HOP_PREFERENCE; // clear persisted route + clearRouteHealth(to); // clear RAM health + return std::nullopt; + } + // We are careful not to return the relay node as the next hop if (node->next_hop != relay_node) { - // LOG_DEBUG("Next hop for 0x%x is 0x%x", to, node->next_hop); - return node->next_hop; + // M1/M2: only emit a stored next_hop if its last byte still maps to a UNIQUE, currently + // reachable direct neighbor. On a dense mesh the last byte collides, so an ambiguous byte + // would unicast a hint toward the wrong physical node; if the neighbor has gone away we'd + // unicast into a void. In both cases flood instead (managed flooding still delivers). + ResolvedNode r = nodeDB->resolveLastByte(node->next_hop, /*requireDirectNeighbor=*/true); + if (r.status == LastByteResolution::Unique) + return node->next_hop; + LOG_WARN("Next hop 0x%x for 0x%x %s; set no pref", node->next_hop, to, + r.status == LastByteResolution::Ambiguous ? "ambiguous among neighbors" : "not a known neighbor"); } else LOG_WARN("Next hop for 0x%x is 0x%x, same as relayer; set no pref", to, node->next_hop); } + +#if HAS_TRAFFIC_MANAGEMENT + // Fallback: TMM overflow cache holds confirmed hops for nodes that have aged out of the hot store. + // It is the same byte source/confidence as NodeInfoLite.next_hop, so it gets the same M1/M2/M3 + // protection: decay a stale/failing route, then only emit a byte that still resolves to a unique + // reachable neighbor. Without this the overflow cache (which holds MORE bytes for MORE nodes) would + // reintroduce exactly the silent-misroute that M1/M2 closes on the hot path. + if (trafficManagementModule) { + uint8_t hint = trafficManagementModule->getNextHopHint(to); + if (hint && hint != relay_node) { + const RouteHealth *h = findRouteHealth(to); + if (h && h->lastNextHop == hint && isRouteStale(*h, millis())) { + LOG_INFO("TMM next hop 0x%x for 0x%x is stale (age/fails); flood and clear", hint, to); + trafficManagementModule->clearNextHop(to); // clear overflow route (setNextHop won't store 0) + clearRouteHealth(to); // clear RAM health + return std::nullopt; + } + ResolvedNode r = nodeDB->resolveLastByte(hint, /*requireDirectNeighbor=*/true); + if (r.status == LastByteResolution::Unique) { + LOG_DEBUG("Next hop for 0x%x is 0x%x (TMM cache)", to, hint); + return hint; + } + LOG_WARN("TMM next hop 0x%x for 0x%x %s; set no pref", hint, to, + r.status == LastByteResolution::Ambiguous ? "ambiguous among neighbors" : "not a known neighbor"); + } + } +#endif + return std::nullopt; } @@ -311,7 +381,10 @@ int32_t NextHopRouter::doRetransmissions() if (!isBroadcast(p.packet->to)) { if (p.numRetransmissions == 1) { - // Last retransmission, reset next_hop (fallback to FloodingRouter) + // Last retransmission: this directed delivery went un-ACKed. Record the failure + // (M3 — accumulates across DMs to age out a flapping/dead route) and reset + // next_hop so the final try falls back to FloodingRouter. + noteRouteFailure(p.packet->to); p.packet->next_hop = NO_NEXT_HOP_PREFERENCE; // Also reset it in the nodeDB meshtastic_NodeInfoLite *sentTo = nodeDB->getMeshNode(p.packet->to); @@ -319,9 +392,32 @@ int32_t NextHopRouter::doRetransmissions() LOG_INFO("Resetting next hop for packet with dest 0x%x\n", p.packet->to); sentTo->next_hop = NO_NEXT_HOP_PREFERENCE; } +#if HAS_TRAFFIC_MANAGEMENT + if (trafficManagementModule) { + trafficManagementModule->clearNextHop(p.packet->to); + } +#endif FloodingRouter::send(packetPool.allocCopy(*p.packet)); } else { +#if NEXTHOP_EARLY_FLOOD_ON_UNVERIFIED + // M4 (gated): if the route isn't proven healthy, don't spend a second directed + // attempt — start flooding one retry sooner to cut recovery latency. A verified + // route (fresh, zero recent failures) keeps the unchanged directed-retry path so + // the sparse-mesh happy path is untouched. + RouteHealth *h = findRouteHealth(p.packet->to); + bool verified = h && h->consecutiveFailures == 0 && !isRouteStale(*h, now); + if (!verified) { + p.packet->next_hop = NO_NEXT_HOP_PREFERENCE; + meshtastic_NodeInfoLite *sentTo = nodeDB->getMeshNode(p.packet->to); + if (sentTo) + sentTo->next_hop = NO_NEXT_HOP_PREFERENCE; + FloodingRouter::send(packetPool.allocCopy(*p.packet)); + } else { + NextHopRouter::send(packetPool.allocCopy(*p.packet)); + } +#else NextHopRouter::send(packetPool.allocCopy(*p.packet)); +#endif } } else { // Note: we call the superclass version because we don't want to have our version of send() add a new @@ -355,3 +451,96 @@ void NextHopRouter::setNextTx(PendingPacket *pending) printPacket("", pending->packet); setReceivedMessage(); // Run ASAP, so we can figure out our correct sleep time } + +// --------------------------------------------------------------------------- +// M3: RAM route-health table. Bounded array with reuse-oldest eviction (same discipline as +// PacketHistory). All age comparisons use unsigned subtraction so they survive the 49.7-day millis() +// rollover. dest == 0 marks an empty slot; learnedAtMsec is normalized to 1 on write so an occupied +// slot is never read as infinitely old. +// --------------------------------------------------------------------------- + +RouteHealth *NextHopRouter::findRouteHealth(NodeNum dest) +{ + if (dest == 0) + return nullptr; + for (auto &h : routeHealth) + if (h.dest == dest) + return &h; + return nullptr; +} + +RouteHealth *NextHopRouter::getOrAllocRouteHealth(NodeNum dest, uint32_t now) +{ + if (dest == 0) + return nullptr; + + RouteHealth *oldest = &routeHealth[0]; + RouteHealth *freeSlot = nullptr; + for (auto &h : routeHealth) { + if (h.dest == dest) + return &h; // existing record + if (h.dest == 0) { + if (!freeSlot) + freeSlot = &h; // remember the first free slot; prefer it over evicting + continue; + } + // Track the oldest occupied slot in case the table is full (rollover-safe). + if ((uint32_t)(now - h.learnedAtMsec) > (uint32_t)(now - oldest->learnedAtMsec)) + oldest = &h; + } + // Claim the free slot if there is one, else reuse the oldest. Reset before use and stamp the dest + // so the record is findable. + RouteHealth *slot = freeSlot ? freeSlot : oldest; + *slot = RouteHealth{}; + slot->dest = dest; + return slot; +} + +void NextHopRouter::noteRouteLearned(NodeNum dest, uint8_t nextHop, uint32_t now) +{ + if (dest == 0 || nextHop == NO_NEXT_HOP_PREFERENCE) + return; + RouteHealth *h = getOrAllocRouteHealth(dest, now); + if (!h) + return; + // A genuinely new next hop earns a clean slate; re-learning the SAME hop keeps the accumulated + // failure count so an asymmetric reverse path that keeps re-teaching a dead forward hop still ages + // out instead of resetting the counter every time. + if (h->lastNextHop != nextHop) { + h->lastNextHop = nextHop; + h->consecutiveFailures = 0; + } + h->learnedAtMsec = now ? now : 1; +} + +void NextHopRouter::noteRouteSuccess(NodeNum dest, uint32_t now) +{ + RouteHealth *h = findRouteHealth(dest); + if (!h) + return; // only routes we actually learned have health to refresh + h->consecutiveFailures = 0; + h->learnedAtMsec = now ? now : 1; +} + +void NextHopRouter::noteRouteFailure(NodeNum dest) +{ + RouteHealth *h = findRouteHealth(dest); + if (!h) + return; // nothing to penalize (we were flooding, or never learned a route here) + if (h->consecutiveFailures < 255) + h->consecutiveFailures++; +} + +bool NextHopRouter::isRouteStale(const RouteHealth &h, uint32_t now) const +{ + if (h.consecutiveFailures >= ROUTE_FAILURE_THRESHOLD) + return true; + return (uint32_t)(now - h.learnedAtMsec) >= ROUTE_TTL_MSEC; +} + +void NextHopRouter::clearRouteHealth(NodeNum dest) +{ + RouteHealth *h = findRouteHealth(dest); + if (h) + *h = RouteHealth{}; +} diff --git a/src/mesh/NextHopRouter.h b/src/mesh/NextHopRouter.h index 42ef13cd9..467d9ca68 100644 --- a/src/mesh/NextHopRouter.h +++ b/src/mesh/NextHopRouter.h @@ -43,6 +43,28 @@ struct PendingPacket { explicit PendingPacket(meshtastic_MeshPacket *p, uint8_t numRetransmissions); }; +/** + * RAM-only per-destination route health. Tracks how fresh a learned next_hop is and how many + * consecutive directed deliveries to it have failed, so getNextHop() can proactively decay a stale or + * repeatedly-failing route back to flooding instead of trusting a dead hop on the next (and on dense + * meshes, slowest) attempt. Not persisted: the learned next_hop itself lives in NodeInfoLite; this is + * just freshness/failure metadata. + */ +struct RouteHealth { + NodeNum dest = 0; ///< destination this record describes; 0 == empty slot + uint32_t learnedAtMsec = 0; ///< millis() when next_hop was last (re)learned (rollover-aware) + uint8_t consecutiveFailures = 0; ///< directed deliveries to `dest` that went un-ACKed + uint8_t lastNextHop = NO_NEXT_HOP_PREFERENCE; ///< the relay byte this health refers to +}; + +// M4 (optional, off by default): when a route is not proven healthy, fall back to flooding one retry +// earlier instead of spending a second directed attempt. Trades airtime for recovery latency on dense +// meshes; leaves the sparse-mesh happy path (fresh, verified routes) unchanged. Measure on the +// simulator before enabling broadly. +#ifndef NEXTHOP_EARLY_FLOOD_ON_UNVERIFIED +#define NEXTHOP_EARLY_FLOOD_ON_UNVERIFIED 0 +#endif + class GlobalPacketIdHashFunction { public: @@ -92,12 +114,22 @@ class NextHopRouter : public FloodingRouter // The number of retransmissions the original sender will do constexpr static uint8_t NUM_RELIABLE_RETX = 3; + // M3: bounded RAM route-health table (reuse-oldest eviction, like PacketHistory) + constexpr static uint8_t ROUTE_HEALTH_MAX = 32; // ~12B/slot -> ~384B + constexpr static uint32_t ROUTE_TTL_MSEC = 30UL * 60 * 1000; // re-discover a route unconfirmed for 30 min + constexpr static uint8_t ROUTE_FAILURE_THRESHOLD = 3; // consecutive un-ACKed directed deliveries -> dead + protected: /** * Pending retransmissions */ std::unordered_map pending; + /** + * Per-destination route health (M3). Bounded array, reuse-oldest eviction. RAM-only. + */ + RouteHealth routeHealth[ROUTE_HEALTH_MAX] = {}; + /** * Should this incoming filter be dropped? * @@ -142,13 +174,38 @@ class NextHopRouter : public FloodingRouter void setNextTx(PendingPacket *pending); + // --- M3 route-health helpers (RAM-only). Protected so ReliableRouter (a subclass) can record + // delivery success, and so the unit-test shim can reach them via `using`. All take `now` where + // time matters so the decay logic is pure and testable without a clock mock. --- + + /// @return the health record for `dest`, or nullptr if we hold none. + RouteHealth *findRouteHealth(NodeNum dest); + /// @return an existing record for `dest`, else a freshly claimed slot (reuse-oldest on overflow). + RouteHealth *getOrAllocRouteHealth(NodeNum dest, uint32_t now); + /// Record that we (re)learned `nextHop` for `dest`. Resets the failure count only when the hop + /// changed (so a flapping reverse-path re-learn of the same dead hop still ages out). + void noteRouteLearned(NodeNum dest, uint8_t nextHop, uint32_t now); + /// Record an end-to-end delivery success to `dest` (clears failures, refreshes freshness). + void noteRouteSuccess(NodeNum dest, uint32_t now); + /// Record that a directed delivery to `dest` went un-ACKed (no-op if we hold no record). + void noteRouteFailure(NodeNum dest); + /// @return true if the route is too old (TTL) or has failed too many times in a row. + bool isRouteStale(const RouteHealth &h, uint32_t now) const; + /// Forget any health record for `dest`. + void clearRouteHealth(NodeNum dest); + +#ifdef PIO_UNIT_TESTING + public: // expose getNextHop to the test shim without widening production visibility +#else private: +#endif /** * Get the next hop for a destination, given the relay node * @return the node number of the next hop, 0 if no preference (fallback to FloodingRouter) */ std::optional getNextHop(NodeNum to, uint8_t relay_node); + private: /** Check if we should be rebroadcasting this packet if so, do so. * @return true if we did rebroadcast */ bool perhapsRebroadcast(const meshtastic_MeshPacket *p) override; diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index 38025ad10..7e5625940 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -1145,6 +1145,20 @@ void NodeDB::initConfigIntervals() #endif } +// Always-on traffic management defaults. Only booleans are written; every +// numeric field stays 0 and resolves to its default_traffic_mgmt_* macro at +// use (e.g. position dedup precision/interval), so fork-wide tuning changes +// take effect without another migration. Rate limiting and the features that +// exhaust or reshape relayed traffic (exhaust_hop_*, drop_unknown_enabled, +// nodeinfo_direct_response) stay opt-in. +static void installTrafficManagementDefaults(meshtastic_LocalModuleConfig &mc) +{ + mc.has_traffic_management = true; + mc.traffic_management = meshtastic_ModuleConfig_TrafficManagementConfig_init_zero; + mc.traffic_management.enabled = true; + mc.traffic_management.position_dedup_enabled = true; +} + void NodeDB::installDefaultModuleConfig() { LOG_INFO("Install default ModuleConfig"); @@ -1262,6 +1276,8 @@ void NodeDB::installDefaultModuleConfig() moduleConfig.has_neighbor_info = true; moduleConfig.neighbor_info.enabled = false; + installTrafficManagementDefaults(moduleConfig); + moduleConfig.has_detection_sensor = true; moduleConfig.detection_sensor.enabled = false; moduleConfig.detection_sensor.detection_trigger_type = meshtastic_ModuleConfig_DetectionSensorConfig_TriggerType_LOGIC_HIGH; @@ -1613,6 +1629,25 @@ bool NodeDB::enforceSatelliteCaps() return trimmedAny; } +// Classify an evicted node's hop-protected category for the warm tier. Favorite/ignored/ +// verified are local flags (rarely reach warm — they're eviction-protected — but classify +// them if they do); otherwise tracker/sensor/tak_tracker are role-protected. +static uint8_t warmProtectedCategory(const meshtastic_NodeInfoLite &n) +{ + if (n.bitfield & (NODEINFO_BITFIELD_IS_FAVORITE_MASK | NODEINFO_BITFIELD_IS_IGNORED_MASK | + NODEINFO_BITFIELD_IS_KEY_MANUALLY_VERIFIED_MASK)) + return static_cast(WarmProtected::Flag); + if (IS_ONE_OF(n.role, meshtastic_Config_DeviceConfig_Role_TRACKER, meshtastic_Config_DeviceConfig_Role_SENSOR, + meshtastic_Config_DeviceConfig_Role_TAK_TRACKER)) + return static_cast(WarmProtected::Role); + return static_cast(WarmProtected::None); +} + +// The warm tier packs the device role into a 4-bit field (WARM_ROLE_MASK). Fail the build +// loudly if a new role outgrows it, rather than silently truncating role on eviction. +static_assert(_meshtastic_Config_DeviceConfig_Role_MAX <= WARM_ROLE_MASK, + "device role no longer fits the 4-bit warm metadata field"); + void NodeDB::cleanupMeshDB() { int newPos = 0, removed = 0; @@ -1639,7 +1674,7 @@ void NodeDB::cleanupMeshDB() // Keep any key we learned (e.g. via a DM before the NodeInfo // exchange completed) rather than losing it with the purge. if (n.public_key.size == 32) - warmStore.absorb(gone, n.last_heard, n.public_key.bytes); + warmStore.absorb(gone, n.last_heard, n.public_key.bytes, n.role, warmProtectedCategory(n)); #endif eraseNodeSatellites(gone); @@ -1822,7 +1857,8 @@ void NodeDB::demoteOldestHotNodesToWarm() continue; // Keep the public key if we have one (40 B warm record); keyless nodes // still get a placeholder so re-admission restores last_heard. - warmStore.absorb(n.num, n.last_heard, n.public_key.size > 0 ? n.public_key.bytes : nullptr); + warmStore.absorb(n.num, n.last_heard, n.public_key.size > 0 ? n.public_key.bytes : nullptr, n.role, + warmProtectedCategory(n)); // Demotion drops the node from the header table, so drop its satellites // too (the eviction chokepoint) — they'd otherwise orphan until the next // enforceSatelliteCaps pass. @@ -2226,6 +2262,16 @@ void NodeDB::loadFromDisk() } } + // Always-on traffic management: a device that has NEVER configured TMM + // (has_traffic_management false — AdminModule always sets the has_ flag on + // write, even when disabling) gets the fork defaults. Explicitly configured + // devices keep their exact settings. + if (!moduleConfig.has_traffic_management) { + LOG_INFO("Traffic management never configured, installing always-on defaults"); + installTrafficManagementDefaults(moduleConfig); + saveToDisk(SEGMENT_MODULECONFIG); + } + state = loadProto(channelFileName, meshtastic_ChannelFile_size, sizeof(meshtastic_ChannelFile), &meshtastic_ChannelFile_msg, &channelFile); if (state != LoadFileResult::LOAD_SUCCESS) { @@ -3295,6 +3341,73 @@ meshtastic_NodeInfoLite *NodeDB::getMeshNode(NodeNum n) return NULL; } +ResolvedNode NodeDB::resolveLastByte(uint8_t lastByte, bool requireDirectNeighbor) +{ + ResolvedNode result; // defaults to {None, 0} + + // 0 is the NO_RELAY_NODE / NO_NEXT_HOP_PREFERENCE sentinel (also what MQTT-sourced packets carry + // when hop_start==0). getLastByteOfNodeNum() never yields 0, so nothing can legitimately match. + if (lastByte == 0) + return result; + + const NodeNum self = getNodeNum(); + NodeNum firstMatch = 0; + uint8_t matches = 0; + + for (size_t i = 0; i < numMeshNodes; i++) { + const meshtastic_NodeInfoLite *node = &meshNodes->at(i); + + // Candidate gate: never resolve to ourselves, the sentinels, or an ignored node. + if (node->num == self || node->num == 0 || node->num == NODENUM_BROADCAST) + continue; + if (nodeInfoLiteIsIgnored(node)) + continue; + if (getLastByteOfNodeNum(node->num) != lastByte) // cheapest discriminator last + continue; + + // Relevance gate: is this node a plausible relay for the requested scope? + bool relevant; + if (requireDirectNeighbor) { + relevant = node->has_hops_away && node->hops_away == 0 && sinceLastSeen(node) < NEXTHOP_NEIGHBOR_FRESH_SECS; + } else { + const bool directNeighbor = node->has_hops_away && node->hops_away == 0; + const bool routerRole = + IS_ONE_OF(node->role, meshtastic_Config_DeviceConfig_Role_ROUTER, meshtastic_Config_DeviceConfig_Role_ROUTER_LATE, + meshtastic_Config_DeviceConfig_Role_CLIENT_BASE); + relevant = directNeighbor || nodeInfoLiteIsFavorite(node) || routerRole; + } + if (!relevant) + continue; + + if (++matches == 1) { + firstMatch = node->num; + } else { + // A second relevant candidate shares this byte: ambiguous. No further scanning can + // change that, so stop early and report the collision. + result.status = LastByteResolution::Ambiguous; + result.num = 0; + return result; + } + } + + if (matches == 1) { + result.status = LastByteResolution::Unique; + result.num = firstMatch; + } + return result; +} + +bool NodeDB::resolveUniqueLastByte(uint8_t lastByte, bool requireDirectNeighbor, NodeNum *outNum) +{ + ResolvedNode r = resolveLastByte(lastByte, requireDirectNeighbor); + if (r.status == LastByteResolution::Unique) { + if (outNum) + *outNum = r.num; + return true; + } + return false; +} + // returns true if the maximum number of nodes is reached or we are running low on memory bool NodeDB::isFull() { @@ -3365,8 +3478,8 @@ meshtastic_NodeInfoLite *NodeDB::getOrCreateMeshNode(NodeNum n) #if WARM_NODE_COUNT > 0 // Demote to the warm tier so the identity (and crucially the // PKI key) outlives the hot-store slot. - warmStore.absorb(evicted.num, evicted.last_heard, - evicted.public_key.size == 32 ? evicted.public_key.bytes : NULL); + warmStore.absorb(evicted.num, evicted.last_heard, evicted.public_key.size == 32 ? evicted.public_key.bytes : NULL, + evicted.role, warmProtectedCategory(evicted)); #endif eraseNodeSatellites(evicted.num); // Shove the remaining nodes down the chain @@ -3395,7 +3508,10 @@ meshtastic_NodeInfoLite *NodeDB::getOrCreateMeshNode(NodeNum n) // Re-admission: restore what the warm tier kept for this node WarmNodeEntry warm; if (warmStore.take(n, warm)) { - lite->last_heard = warm.last_heard; + lite->last_heard = warmTimeOf(warm); // mask off the stolen role/protected metadata bits + // Restore the role the warm tier cached, so re-admission isn't stuck at CLIENT + // until the next NodeInfo arrives. + lite->role = static_cast(warmRoleOf(warm)); if (!memfll(warm.public_key, 0, sizeof(warm.public_key))) { lite->public_key.size = 32; memcpy(lite->public_key.bytes, warm.public_key, 32); diff --git a/src/mesh/NodeDB.h b/src/mesh/NodeDB.h index f2df5ef1d..fa9c886f7 100644 --- a/src/mesh/NodeDB.h +++ b/src/mesh/NodeDB.h @@ -115,6 +115,20 @@ uint32_t sinceLastSeen(const meshtastic_NodeInfoLite *n); /// Given a packet, return how many seconds in the past (vs now) it was received uint32_t sinceReceived(const meshtastic_MeshPacket *p); +/// Outcome of mapping a single on-wire last-byte (next_hop / relay_node) back to a full NodeNum. +/// Because the wire only carries the last byte of a 32-bit node number, the mapping is ambiguous on +/// dense meshes (the "birthday problem"). Callers must treat Ambiguous and None as "don't trust it". +enum class LastByteResolution : uint8_t { + None, ///< no relevant candidate node has this last byte + Unique, ///< exactly one relevant candidate -> `num` is valid + Ambiguous, ///< two or more relevant candidates collide on this byte +}; + +struct ResolvedNode { + LastByteResolution status = LastByteResolution::None; + NodeNum num = 0; ///< valid only when status == Unique +}; + /// Given a packet, return the number of hops used to reach this node. /// Returns defaultIfUnknown if the number of hops couldn't be determined. int8_t getHopsAway(const meshtastic_MeshPacket &p, int8_t defaultIfUnknown = -1); @@ -331,6 +345,23 @@ class NodeDB /// with no allocation side effects (unlike getOrCreateMeshNode). uint32_t hotNodeLastHeard(NodeNum n) const; + /** + * Resolve a single on-wire last-byte (e.g. next_hop / relay_node) back to a unique full NodeNum, + * detecting last-byte collisions instead of silently picking the first match. A 1-byte id only + * needs to be unique among a node's plausible relays, not the whole mesh, so we scope the search: + * - requireDirectNeighbor == true : candidates are direct neighbors (hops_away==0) heard within + * NEXTHOP_NEIGHBOR_FRESH_SECS. Use on the SEND path. + * - requireDirectNeighbor == false : also accept favorites and router-role nodes (unknown hop + * distance allowed). Use when learning / preserving hops. + * Ignored nodes, our own node, and the broadcast/0 sentinels are never candidates. On a tie the + * result is Ambiguous (no tie-break) so callers fall back to flooding rather than misroute. + */ + ResolvedNode resolveLastByte(uint8_t lastByte, bool requireDirectNeighbor); + + /// Convenience wrapper around resolveLastByte(): true iff exactly one relevant candidate matches. + /// Ambiguous and None both return false (the safe answer for learning / hop preservation). + bool resolveUniqueLastByte(uint8_t lastByte, bool requireDirectNeighbor, NodeNum *outNum = nullptr); + // Thread-safe satellite-map accessors. Return false if absent or the // corresponding DB is compiled out. bool copyNodePosition(NodeNum n, meshtastic_PositionLite &out) const; diff --git a/src/mesh/PacketHistory.cpp b/src/mesh/PacketHistory.cpp index f5c36b083..e4f565d1a 100644 --- a/src/mesh/PacketHistory.cpp +++ b/src/mesh/PacketHistory.cpp @@ -486,7 +486,11 @@ bool PacketHistory::wasRelayer(const uint8_t relayer, const uint32_t id, const N } /* Check if a certain node was a relayer of a packet in the history given iterator - * @return true if node was indeed a relayer, false if not */ + * @return true if node was indeed a relayer, false if not + * NOTE: intentionally byte-domain. Both `relayer` and relayed_by[] are on-wire last bytes, so this + * answers "did a relayer with this byte touch the packet" — correct without resolving to a NodeNum. + * The collision risk is neutralized where the result is consumed (route learning in + * NextHopRouter::sniffReceived now gates the write through NodeDB::resolveUniqueLastByte). */ bool PacketHistory::wasRelayer(const uint8_t relayer, const PacketRecord &r, bool *wasSole) { bool found = false; diff --git a/src/mesh/ReliableRouter.cpp b/src/mesh/ReliableRouter.cpp index a1cfb1200..3126c820e 100644 --- a/src/mesh/ReliableRouter.cpp +++ b/src/mesh/ReliableRouter.cpp @@ -151,6 +151,10 @@ void ReliableRouter::sniffReceived(const meshtastic_MeshPacket *p, const meshtas LOG_DEBUG("Received a %s for 0x%x, stopping retransmissions", ackId ? "ACK" : "NAK", ackId); if (ackId) { stopRetransmission(p->to, ackId); + // M3: an end-to-end ACK proves the directed route to the ACK's sender currently works, + // so clear its failure count and refresh freshness (keeps a good route pinned). + if (!isBroadcast(getFrom(p))) + noteRouteSuccess(getFrom(p), millis()); } else { stopRetransmission(p->to, nakId); } diff --git a/src/mesh/Router.cpp b/src/mesh/Router.cpp index 9b3aa4676..50d0f2bcc 100644 --- a/src/mesh/Router.cpp +++ b/src/mesh/Router.cpp @@ -114,37 +114,24 @@ bool Router::shouldDecrementHopLimit(const meshtastic_MeshPacket *p) } #endif - // For subsequent hops, check if previous relay is a favorite router - // Optimized search for favorite routers with matching last byte - // Check ordering optimized for IoT devices (cheapest checks first) - for (size_t i = 0; i < nodeDB->getNumMeshNodes(); i++) { - meshtastic_NodeInfoLite *node = nodeDB->getMeshNodeByIndex(i); - if (!node) - continue; - - // Check 1: is_favorite (cheapest - single bit test) - if (!nodeInfoLiteIsFavorite(node)) - continue; - - // Check 2: has_user (cheap - single bit test) - if (!nodeInfoLiteHasUser(node)) - continue; - - // Check 3: role check (moderate cost - multiple comparisons) - if (!IS_ONE_OF(node->role, meshtastic_Config_DeviceConfig_Role_ROUTER, meshtastic_Config_DeviceConfig_Role_ROUTER_LATE, - meshtastic_Config_DeviceConfig_Role_CLIENT_BASE)) { - continue; - } - - // Check 4: last byte extraction and comparison (most expensive) - if (nodeDB->getLastByteOfNodeNum(node->num) == p->relay_node) { - // Found a favorite router match - LOG_DEBUG("Identified favorite relay router 0x%x from last byte 0x%x", node->num, p->relay_node); + // For subsequent hops, preserve hop_limit only when the previous relay is UNAMBIGUOUSLY a favorite + // router. The relay_node byte is just the last byte of a 32-bit node number, so on a dense mesh it + // collides; the old "first matching node wins" scan could preserve hops for the wrong node + // (non-deterministic, depends on NodeDB order). resolveLastByte() reports a collision instead, and + // we re-check the favorite/router predicate on the single resolved node. On ambiguity/none we + // decrement (the safe default). + NodeNum resolved = 0; + if (nodeDB->resolveUniqueLastByte(p->relay_node, /*requireDirectNeighbor=*/false, &resolved)) { + const meshtastic_NodeInfoLite *node = nodeDB->getMeshNode(resolved); + if (node && nodeInfoLiteIsFavorite(node) && nodeInfoLiteHasUser(node) && + IS_ONE_OF(node->role, meshtastic_Config_DeviceConfig_Role_ROUTER, meshtastic_Config_DeviceConfig_Role_ROUTER_LATE, + meshtastic_Config_DeviceConfig_Role_CLIENT_BASE)) { + LOG_DEBUG("Identified unique favorite relay router 0x%x from last byte 0x%x", resolved, p->relay_node); return false; // Don't decrement hop_limit } } - // No favorite router match found, decrement hop_limit + // No unambiguous favorite router match found, decrement hop_limit return true; } diff --git a/src/mesh/WarmNodeStore.cpp b/src/mesh/WarmNodeStore.cpp index 8617f505c..39fbd90c6 100644 --- a/src/mesh/WarmNodeStore.cpp +++ b/src/mesh/WarmNodeStore.cpp @@ -12,7 +12,12 @@ #if defined(NRF52840_XXAA) #include "flash/flash_nrf5x.h" -#define WARM_RING_MAGIC 0x474E5257u // "WRNG" +#define WARM_RING_MAGIC 0x324E5257u // "WRN2" — v2: last_heard low bits carry role + protected category +#define WARM_RING_MAGIC_V1 0x474E5257u // "WRNG" — v1: last_heard was a plain timestamp. +// v1 pages are still read on upgrade: we keep each record's identity + public key but +// DISCARD its last_heard (the old timestamp would be misread as role/protected bits). +// Records re-rank and re-learn their role on the next contact. Legacy pages convert to +// v2 naturally as the ring rotates. // A tombstone is an entry record whose last_heard is all-ones — getTime() // (unix seconds) cannot reach 0xFFFFFFFF until 2106, and erased flash is // detected via num == 0xFFFFFFFF before last_heard is ever inspected. @@ -28,7 +33,10 @@ struct WarmStoreHeader { }; static_assert(sizeof(WarmStoreHeader) == 16, "header layout is part of the persistence format"); -#define WARM_STORE_MAGIC 0x314D5257u // "WRM1" +#define WARM_STORE_MAGIC 0x324D5257u // "WRM2" — v2: last_heard low bits carry role + protected category +#define WARM_STORE_MAGIC_V1 \ + 0x314D5257u // "WRM1" — v1: last_heard was a plain timestamp. On upgrade we keep + // identity + key but discard last_heard, then rewrite as v2. #ifdef FSCom static const char *warmFileName = "/prefs/warm.dat"; @@ -96,11 +104,13 @@ WarmNodeEntry *WarmNodeStore::place(NodeNum num, uint32_t lastHeard, const uint8 slot = &e; break; } + // Compare on the time bits only — the low metadata bits (role/protected) must + // not perturb LRU victim selection. if (keyIsSet(e.public_key)) { - if (!oldestKeyed || e.last_heard < oldestKeyed->last_heard) + if (!oldestKeyed || warmTimeOf(e) < warmTimeOf(*oldestKeyed)) oldestKeyed = &e; } else { - if (!oldestKeyless || e.last_heard < oldestKeyless->last_heard) + if (!oldestKeyless || warmTimeOf(e) < warmTimeOf(*oldestKeyless)) oldestKeyless = &e; } } @@ -121,14 +131,28 @@ WarmNodeEntry *WarmNodeStore::place(NodeNum num, uint32_t lastHeard, const uint8 return slot; } -bool WarmNodeStore::absorb(NodeNum num, uint32_t lastHeard, const uint8_t *key32) +bool WarmNodeStore::absorb(NodeNum num, uint32_t lastHeard, const uint8_t *key32, uint8_t role, uint8_t protectedCat) { - const WarmNodeEntry *slot = place(num, lastHeard, key32); + // Pack role + protected category into the low bits of last_heard. place() and ring + // replay store the raw word verbatim, so the metadata round-trips through flash. + const uint32_t packed = warmPackLastHeard(lastHeard, role, protectedCat); + const WarmNodeEntry *slot = place(num, packed, key32); if (!slot) return false; persistEntry(*slot); - LOG_MIGRATION("WarmStore absorb 0x%08x key=%d last_heard=%u (now %u/%u)", (unsigned)num, keyIsSet(slot->public_key) ? 1 : 0, - (unsigned)lastHeard, (unsigned)count(), (unsigned)capacity()); + LOG_MIGRATION("WarmStore absorb 0x%08x key=%d last_heard=%u role=%u prot=%u (now %u/%u)", (unsigned)num, + keyIsSet(slot->public_key) ? 1 : 0, (unsigned)warmTimeOf(*slot), (unsigned)role, (unsigned)protectedCat, + (unsigned)count(), (unsigned)capacity()); + return true; +} + +bool WarmNodeStore::lookupMeta(NodeNum num, uint8_t &role, uint8_t &protectedCat) const +{ + const WarmNodeEntry *e = find(num); + if (!e) + return false; + role = warmRoleOf(*e); + protectedCat = warmProtOf(*e); return true; } @@ -231,10 +255,22 @@ bool WarmNodeStore::saveIfDirty() // (stranded live entries re-appended, then erased). Flash access holds spiLock — // the page cache is shared with InternalFS/LittleFS. -bool WarmNodeStore::ringReadHeader(uint8_t page, WarmPageHeader &h) const +bool WarmNodeStore::ringReadHeader(uint8_t page, WarmPageHeader &h, bool *legacy) const { flash_nrf5x_read(&h, WARM_FLASH_PAGE_ADDR(page), sizeof(h)); - return h.magic == WARM_RING_MAGIC && h.seq != 0xFFFFFFFFu; + if (h.seq == 0xFFFFFFFFu) + return false; // erased page + if (h.magic == WARM_RING_MAGIC) { + if (legacy) + *legacy = false; + return true; + } + if (h.magic == WARM_RING_MAGIC_V1) { + if (legacy) + *legacy = true; // v1 page: replay it, but discard last_heard (see WARM_RING_MAGIC_V1) + return true; + } + return false; } // Caller holds spiLock. @@ -347,11 +383,13 @@ void WarmNodeStore::load() // Order valid pages by ascending seq so replay applies oldest first uint8_t order[WARM_FLASH_PAGES] = {}; uint32_t seqs[WARM_FLASH_PAGES] = {}; + bool legacyOf[WARM_FLASH_PAGES] = {}; // per-page: v1 (WRNG) → discard last_heard on replay uint8_t nValid = 0; uint8_t nCorrupt = 0; for (uint8_t p = 0; p < WARM_FLASH_PAGES; p++) { WarmPageHeader h; - if (!ringReadHeader(p, h)) { + bool legacy = false; + if (!ringReadHeader(p, h, &legacy)) { // An erased page reads back all-ones; any other magic is a // partially-written or bit-rotted header we're dropping, so flag it // rather than silently treating the loss as a clean empty ring. @@ -359,6 +397,7 @@ void WarmNodeStore::load() nCorrupt++; continue; } + legacyOf[p] = legacy; uint8_t pos = nValid; while (pos > 0 && static_cast(h.seq - seqs[pos - 1]) < 0) { order[pos] = order[pos - 1]; @@ -382,8 +421,10 @@ void WarmNodeStore::load() } uint32_t replayed = 0; + uint32_t migrated = 0; for (uint8_t k = 0; k < nValid; k++) { const uint8_t p = order[k]; + const bool legacy = legacyOf[p]; uint16_t slot = 0; for (; slot < kRecordsPerPage; slot++) { WarmNodeEntry rec; @@ -400,7 +441,14 @@ void WarmNodeStore::load() memset(e, 0, sizeof(*e)); } } else { - const WarmNodeEntry *e = place(rec.num, rec.last_heard, rec.public_key); + // v1 (legacy) record: keep identity + key, but discard the old timestamp — + // its low bits would otherwise be misread as role/protected metadata. + uint32_t lh = rec.last_heard; + if (legacy) { + lh = 0; + migrated++; + } + const WarmNodeEntry *e = place(rec.num, lh, rec.public_key); if (e) pageOf[e - entries] = p; } @@ -409,10 +457,17 @@ void WarmNodeStore::load() activePage = p; writeSlot = slot; nextSeq = seqs[k] + 1; + // If the head is a v1 page, force the next append to rotate into a fresh v2 page, + // so new (v2) records never land in a page whose header says v1 (which would make + // a later load discard their last_heard — including the role/protected we just set). + if (legacy) + writeSlot = kRecordsPerPage; } } if (nCorrupt) LOG_WARN("WarmStore: dropped %u corrupt ring page(s), some nodes lost", nCorrupt); + if (migrated) + LOG_INFO("WarmStore: migrated %u v1 record(s) (kept key, discarded last_heard)", (unsigned)migrated); LOG_INFO("WarmStore: replayed %u ring records -> %u live nodes (page %u, slot %u)", (unsigned)replayed, (unsigned)count(), activePage, writeSlot); } @@ -479,7 +534,10 @@ void WarmNodeStore::load() LOG_WARN("WarmStore: %s header read failed, starting empty", warmFileName); return; } - if (h.magic != WARM_STORE_MAGIC || h.entrySize != sizeof(WarmNodeEntry) || h.count > WARM_NODE_COUNT) { + // v1 (WRM1) is still accepted: same record size, but its last_heard was a plain + // timestamp. We keep identity + key and discard last_heard on load (see below). + const bool legacy = (h.magic == WARM_STORE_MAGIC_V1); + if ((h.magic != WARM_STORE_MAGIC && !legacy) || h.entrySize != sizeof(WarmNodeEntry) || h.count > WARM_NODE_COUNT) { f.close(); LOG_WARN("WarmStore: %s header invalid (magic=0x%08x entrySize=%u count=%u), starting empty", warmFileName, h.magic, h.entrySize, h.count); @@ -493,14 +551,25 @@ void WarmNodeStore::load() LOG_WARN("WarmStore: %s entries read failed, starting empty", warmFileName); return; } + // CRC covers the bytes as written (v1 still has the old last_heard), so check before migrating. if (crc32Buffer(entries, len) != h.crc) { LOG_WARN("WarmStore: %s CRC mismatch, starting empty", warmFileName); + memset(entries, 0, WARM_NODE_COUNT * sizeof(WarmNodeEntry)); return; } + if (legacy) { + // Migrate v1 → v2: discard the old last_heard (its low bits would be misread as + // role/protected); keep num + public_key. Mark dirty so save() rewrites as v2. + for (size_t i = 0; i < WARM_NODE_COUNT; i++) + if (entries[i].num) + entries[i].last_heard = 0; + dirty = true; + } } else { f.close(); } - LOG_INFO("WarmStore: loaded %u warm nodes from %s", h.count, warmFileName); + LOG_INFO("WarmStore: loaded %u warm nodes from %s%s", h.count, warmFileName, + legacy ? " (v1 migrated: discarded last_heard)" : ""); } bool WarmNodeStore::save() diff --git a/src/mesh/WarmNodeStore.h b/src/mesh/WarmNodeStore.h index c5320317b..8d9bb5be4 100644 --- a/src/mesh/WarmNodeStore.h +++ b/src/mesh/WarmNodeStore.h @@ -34,11 +34,50 @@ */ struct WarmNodeEntry { NodeNum num; // 0 = empty slot - uint32_t last_heard; // recency for LRU ordering + uint32_t last_heard; // recency for LRU ordering — see the metadata steal below uint8_t public_key[32]; // all-zero = no key (a real key is never all-zero) }; static_assert(sizeof(WarmNodeEntry) == 40, "WarmNodeEntry must stay 40 B — persistence format depends on it"); +// Metadata packed into the low bits of last_heard. +// +// The warm tier only uses last_heard to LRU-rank evicted (long-tail) nodes, so ~minute +// recency resolution is plenty. We reclaim the low WARM_META_BITS of that field to carry +// the evicted node's device role + a protected category, at zero cost to record size +// (entry stays 40 B; no RAM/flash growth). The high bits remain a real unix-seconds +// timestamp quantised to (1 << WARM_META_BITS) seconds. +// +// Safe because: a real timestamp can never be all-ones (the tombstone sentinel) before +// 2106, and tombstones/erased flash are detected via num before last_heard is read. Only +// the LOW bits are stolen — the high (era) bits are untouched, so the time range is intact. +static constexpr uint32_t WARM_META_BITS = 6; // role(4) + protected(2) +static constexpr uint32_t WARM_META_MASK = (1u << WARM_META_BITS) - 1; // 0x3F → 64 s quantum +static constexpr uint32_t WARM_TIME_MASK = ~WARM_META_MASK; // 0xFFFFFFC0 +static constexpr uint32_t WARM_ROLE_MASK = 0x0Fu; // bits [3:0] device role (0..12) +static constexpr uint32_t WARM_PROT_SHIFT = 4; // bits [5:4] protected category +static constexpr uint32_t WARM_PROT_MASK = 0x03u; + +// Protected category cached alongside role so consumers needn't re-derive the mapping. +enum class WarmProtected : uint8_t { None = 0, Role = 1, Flag = 2 }; + +inline uint32_t warmPackLastHeard(uint32_t lastHeard, uint8_t role, uint8_t prot) +{ + return (lastHeard & WARM_TIME_MASK) | (static_cast(role) & WARM_ROLE_MASK) | + ((static_cast(prot) & WARM_PROT_MASK) << WARM_PROT_SHIFT); +} +inline uint32_t warmTimeOf(const WarmNodeEntry &e) +{ + return e.last_heard & WARM_TIME_MASK; +} +inline uint8_t warmRoleOf(const WarmNodeEntry &e) +{ + return static_cast(e.last_heard & WARM_ROLE_MASK); +} +inline uint8_t warmProtOf(const WarmNodeEntry &e) +{ + return static_cast((e.last_heard >> WARM_PROT_SHIFT) & WARM_PROT_MASK); +} + // Gated on NRF52840_XXAA: the ring sits at 0xEA000 // valid only on the 1 MB-flash nRF52840. #if defined(NRF52840_XXAA) @@ -58,8 +97,15 @@ class WarmNodeStore /// Remember an evicted hot node. Keyless candidates never displace keyed /// entries; otherwise the oldest (keyless-first) entry is replaced. + /// @param role the node's device role (meshtastic_Config_DeviceConfig_Role, 0..12) + /// @param protectedCat WarmProtected category cached for the hop-trim path /// @return true if the node was stored or updated - bool absorb(NodeNum num, uint32_t lastHeard, const uint8_t *key32 /* may be NULL */); + bool absorb(NodeNum num, uint32_t lastHeard, const uint8_t *key32 /* may be NULL */, uint8_t role = 0, + uint8_t protectedCat = 0); + + /// Look up the cached device role + protected category for a warm node. + /// @return false if the node is not in the warm tier. + bool lookupMeta(NodeNum num, uint8_t &role, uint8_t &protectedCat) const; /// Find and remove an entry (used when the node is re-admitted to the hot store). bool take(NodeNum num, WarmNodeEntry &out); @@ -121,7 +167,7 @@ class WarmNodeStore void ringAppend(const WarmNodeEntry &rec, int storeSlot /* -1 for tombstones */); void ringRotate(); // reclaim oldest page, compacting stranded live entries void ringOpenPage(uint8_t page); // erase + write header (seq = nextSeq++) - bool ringReadHeader(uint8_t page, WarmPageHeader &h) const; + bool ringReadHeader(uint8_t page, WarmPageHeader &h, bool *legacy = nullptr) const; #endif bool save(); diff --git a/src/mesh/mesh-pb-constants.h b/src/mesh/mesh-pb-constants.h index 6f9f4bc91..28dbe3bb7 100644 --- a/src/mesh/mesh-pb-constants.h +++ b/src/mesh/mesh-pb-constants.h @@ -143,10 +143,14 @@ static inline int get_max_num_nodes() #define MAX_NUM_CHANNELS (member_size(meshtastic_ChannelFile, channels) / member_size(meshtastic_ChannelFile, channels[0])) // Traffic Management module configuration -// Enable per-variant by defining HAS_TRAFFIC_MANAGEMENT=1 in variant.h -#ifndef HAS_TRAFFIC_MANAGEMENT +// Enabled by default; STM32WL is excluded due to RAM constraints (MAX_NUM_NODES=10). +// Disable per-variant by defining HAS_TRAFFIC_MANAGEMENT=0 in variant.h +#ifdef ARCH_STM32WL #define HAS_TRAFFIC_MANAGEMENT 0 #endif +#ifndef HAS_TRAFFIC_MANAGEMENT +#define HAS_TRAFFIC_MANAGEMENT 1 +#endif // HopScalingModule - variable hop module: dynamically adjusts broadcast hop_limit based on mesh density // Enable per-variant by defining HAS_VARIABLE_HOPS=1 in variant.h diff --git a/src/modules/TraceRouteModule.cpp b/src/modules/TraceRouteModule.cpp index b6cb5d041..915203ed1 100644 --- a/src/modules/TraceRouteModule.cpp +++ b/src/modules/TraceRouteModule.cpp @@ -8,6 +8,10 @@ #include "meshUtils.h" #include +#if HAS_TRAFFIC_MANAGEMENT +#include "modules/TrafficManagementModule.h" +#endif + extern graphics::Screen *screen; TraceRouteModule *traceRouteModule; @@ -323,6 +327,14 @@ void TraceRouteModule::maybeSetNextHop(NodeNum target, uint8_t nextHopByte) LOG_INFO("Updating next-hop for 0x%08x to 0x%02x based on traceroute", target, nextHopByte); node->next_hop = nextHopByte; } + +#if HAS_TRAFFIC_MANAGEMENT + // Mirror into the TMM overflow cache. Traceroute is the highest-confidence + // source (full known route), and this captures the target even when it isn't + // in the hot NodeDB — same rationale as the ACK-confirmed path in NextHopRouter. + if (trafficManagementModule) + trafficManagementModule->setNextHop(target, nextHopByte); +#endif } void TraceRouteModule::processUpgradedPacket(const meshtastic_MeshPacket &mp) diff --git a/src/modules/TrafficManagementModule.cpp b/src/modules/TrafficManagementModule.cpp index 4c73c2947..a89940f2e 100644 --- a/src/modules/TrafficManagementModule.cpp +++ b/src/modules/TrafficManagementModule.cpp @@ -28,7 +28,6 @@ namespace constexpr uint32_t kMaintenanceIntervalMs = 60 * 1000UL; // Cache cleanup interval constexpr uint32_t kUnknownResetMs = 60 * 1000UL; // Unknown packet window -constexpr uint8_t kMaxCuckooKicks = 16; // Max displacement chain length // NodeInfo direct response: enforced maximum hops by device role // Both use maxHops logic (respond when hopsAway <= threshold) @@ -76,6 +75,21 @@ bool isWithinWindow(uint32_t nowMs, uint32_t startMs, uint32_t intervalMs) return (nowMs - startMs) < intervalMs; } +/** + * Slide an 8-bit relative timestamp back by a wall-clock slab during epoch rebase. + * + * Entries older than the slab clamp to 0 (then reclaimed by the maintenance sweep); + * live entries keep their reconstructed age minus a sub-tick remainder. Each field + * slides by its own resolution's worth of ticks, so a single slab covers all three. + */ +inline void slideRelativeTime(uint8_t &ticks, uint32_t slabMs, uint16_t resolutionSecs) +{ + if (ticks == 0 || resolutionSecs == 0) + return; + uint32_t dec = slabMs / (static_cast(resolutionSecs) * 1000UL); + ticks = (ticks > dec) ? static_cast(ticks - dec) : 0; +} + /** * Truncate lat/lon to specified precision for position deduplication. * @@ -203,27 +217,16 @@ TrafficManagementModule::TrafficManagementModule() : MeshModule("TrafficManageme #endif // TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 #if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - TM_LOG_INFO("Allocating NodeInfo cache: target=%u occupancy=%u%% payload=%u bytes (PSRAM) tags=%u bytes (%u-bit, %u slots, " - "%u buckets x %u)", - static_cast(nodeInfoTargetEntries()), static_cast(nodeInfoTargetOccupancyPercent()), - static_cast(nodeInfoTargetEntries() * sizeof(NodeInfoPayloadEntry)), - static_cast(nodeInfoIndexMetadataBudgetBytes()), static_cast(nodeInfoTagBits()), - static_cast(nodeInfoIndexSlots()), static_cast(nodeInfoBucketCount()), - static_cast(nodeInfoBucketSize())); + TM_LOG_INFO("Allocating NodeInfo cache: %u entries, %u bytes (PSRAM flat array)", + static_cast(nodeInfoTargetEntries()), + static_cast(nodeInfoTargetEntries() * sizeof(NodeInfoPayloadEntry))); - nodeInfoIndex = static_cast(calloc(nodeInfoIndexMetadataBudgetBytes(), sizeof(uint8_t))); - if (!nodeInfoIndex) { - TM_LOG_WARN("NodeInfo index allocation failed; direct responses will fall back to NodeDB"); + nodeInfoPayload = static_cast(ps_calloc(nodeInfoTargetEntries(), sizeof(NodeInfoPayloadEntry))); + if (nodeInfoPayload) { + nodeInfoPayloadFromPsram = true; + TM_LOG_INFO("NodeInfo PSRAM cache ready"); } else { - nodeInfoPayload = static_cast(ps_calloc(nodeInfoTargetEntries(), sizeof(NodeInfoPayloadEntry))); - if (nodeInfoPayload) { - nodeInfoPayloadFromPsram = true; - TM_LOG_INFO("NodeInfo bucketed cuckoo cache ready"); - } else { - TM_LOG_WARN("NodeInfo PSRAM payload allocation failed; direct responses will fall back to NodeDB"); - free(nodeInfoIndex); - nodeInfoIndex = nullptr; - } + TM_LOG_WARN("NodeInfo PSRAM payload allocation failed; direct responses will fall back to NodeDB"); } #else TM_LOG_DEBUG("NodeInfo PSRAM cache not available on this target"); @@ -255,11 +258,6 @@ TrafficManagementModule::~TrafficManagementModule() delete[] nodeInfoPayload; nodeInfoPayload = nullptr; } - - if (nodeInfoIndex) { - free(nodeInfoIndex); - nodeInfoIndex = nullptr; - } } // ============================================================================= @@ -292,17 +290,11 @@ void TrafficManagementModule::incrementStat(uint32_t *field) } // ============================================================================= -// Cuckoo Hash Table Operations +// Flat Unified Cache Operations // ============================================================================= /** - * Find an existing entry for the given node. - * - * Cuckoo hashing guarantees that if an entry exists, it's in one of exactly - * two locations: hash1(node) or hash2(node). This provides O(1) lookup. - * - * @param node NodeNum to search for - * @return Pointer to entry if found, nullptr otherwise + * Find an existing entry for the given node (linear scan). */ TrafficManagementModule::UnifiedCacheEntry *TrafficManagementModule::findEntry(NodeNum node) { @@ -313,35 +305,26 @@ TrafficManagementModule::UnifiedCacheEntry *TrafficManagementModule::findEntry(N if (!cache || node == 0) return nullptr; - // Check primary location - uint16_t h1 = cuckooHash1(node); - if (cache[h1].node == node) - return &cache[h1]; - - // Check alternate location - uint16_t h2 = cuckooHash2(node); - if (cache[h2].node == node) - return &cache[h2]; - + for (uint16_t i = 0; i < cacheSize(); i++) { + if (cache[i].node == node) + return &cache[i]; + } return nullptr; #endif } /** - * Find or create an entry for the given node using cuckoo hashing. + * Find or create an entry for the given node. * - * If the node exists, returns the existing entry. Otherwise, attempts to - * insert a new entry using cuckoo displacement: - * - * 1. Try to insert at h1(node) - if empty, done - * 2. Try to insert at h2(node) - if empty, done - * 3. Kick existing entry from h1 to its alternate location - * 4. Repeat up to kMaxCuckooKicks times - * 5. If cycle detected or max kicks exceeded, evict oldest entry + * One linear pass tracks the match, the first empty slot, and the eviction + * victim. When the cache is full, the victim is the stalest entry (largest + * of its three relative timestamps is smallest), preferring entries without + * a next_hop hint — those hints are the long-tail routing state the cache + * exists to keep, and the maintenance sweep never ages them out. * * @param node NodeNum to find or create * @param isNew Set to true if a new entry was created - * @return Pointer to entry, or nullptr if allocation failed + * @return Pointer to entry, or nullptr if the cache is unavailable */ TrafficManagementModule::UnifiedCacheEntry *TrafficManagementModule::findOrCreateEntry(NodeNum node, bool *isNew) { @@ -351,304 +334,76 @@ TrafficManagementModule::UnifiedCacheEntry *TrafficManagementModule::findOrCreat *isNew = false; return nullptr; #else - if (!cache || node == 0) { - if (isNew) - *isNew = false; + if (isNew) + *isNew = false; + if (!cache || node == 0) return nullptr; - } - // Check if entry already exists (O(1) lookup) - uint16_t h1 = cuckooHash1(node); - if (cache[h1].node == node) { - if (isNew) - *isNew = false; - return &cache[h1]; - } + UnifiedCacheEntry *empty = nullptr; + UnifiedCacheEntry *victim = nullptr; + bool victimHasHop = true; + uint8_t victimRecency = UINT8_MAX; - uint16_t h2 = cuckooHash2(node); - if (cache[h2].node == node) { - if (isNew) - *isNew = false; - return &cache[h2]; - } - - // Entry doesn't exist - try to insert - - // Prefer empty slot at h1 - if (cache[h1].node == 0) { - memset(&cache[h1], 0, sizeof(UnifiedCacheEntry)); - cache[h1].node = node; - if (isNew) - *isNew = true; - return &cache[h1]; - } - - // Try empty slot at h2 - if (cache[h2].node == 0) { - memset(&cache[h2], 0, sizeof(UnifiedCacheEntry)); - cache[h2].node = node; - if (isNew) - *isNew = true; - return &cache[h2]; - } - - // Both slots occupied - perform cuckoo displacement - // Start by kicking entry at h1 to its alternate location - UnifiedCacheEntry displaced = cache[h1]; - memset(&cache[h1], 0, sizeof(UnifiedCacheEntry)); - cache[h1].node = node; - - for (uint8_t kicks = 0; kicks < kMaxCuckooKicks; kicks++) { - // Find alternate location for displaced entry - uint16_t altH1 = cuckooHash1(displaced.node); - uint16_t altH2 = cuckooHash2(displaced.node); - uint16_t altSlot = (altH1 == h1) ? altH2 : altH1; - - if (cache[altSlot].node == 0) { - // Found empty slot - insert displaced entry - cache[altSlot] = displaced; - if (isNew) - *isNew = true; - return &cache[h1]; + for (uint16_t i = 0; i < cacheSize(); i++) { + UnifiedCacheEntry &e = cache[i]; + if (e.node == node) + return &e; + if (e.node == 0) { + if (!empty) + empty = &e; + continue; + } + if (empty) + continue; // an empty slot beats any victim; stop scoring + const bool hasHop = e.next_hop != 0; + uint8_t recency = e.pos_time; + if (e.rate_time > recency) + recency = e.rate_time; + if (e.unknown_time > recency) + recency = e.unknown_time; + if (!victim || (hasHop == victimHasHop ? recency < victimRecency : !hasHop)) { + victim = &e; + victimHasHop = hasHop; + victimRecency = recency; } - - // Kick entry from alternate slot - UnifiedCacheEntry temp = cache[altSlot]; - cache[altSlot] = displaced; - displaced = temp; - h1 = altSlot; } - // Cuckoo cycle detected or max kicks exceeded. - // The displaced entry has no valid cuckoo slot — drop it to preserve cache integrity. - // Placing it at an arbitrary slot would make it unreachable by findEntry(). - TM_LOG_DEBUG("Cuckoo cycle, evicting node 0x%08x", displaced.node); - + UnifiedCacheEntry *slot = empty ? empty : victim; + if (!slot) + return nullptr; + if (!empty) + TM_LOG_DEBUG("Unified cache full, evicting node 0x%08x", slot->node); + memset(slot, 0, sizeof(UnifiedCacheEntry)); + slot->node = node; if (isNew) *isNew = true; - return &cache[cuckooHash1(node)]; + return slot; #endif } const TrafficManagementModule::NodeInfoPayloadEntry *TrafficManagementModule::findNodeInfoEntry(NodeNum node) const { #if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoPayload || !nodeInfoIndex || node == 0) + if (!nodeInfoPayload || node == 0) return nullptr; - uint16_t payloadIndex = findNodeInfoPayloadIndex(node); - if (payloadIndex >= nodeInfoTargetEntries()) - return nullptr; - - return &nodeInfoPayload[payloadIndex]; + for (uint16_t i = 0; i < nodeInfoTargetEntries(); i++) { + if (nodeInfoPayload[i].node == node) + return &nodeInfoPayload[i]; + } + return nullptr; #else (void)node; return nullptr; #endif } -uint16_t TrafficManagementModule::encodeNodeInfoTag(uint16_t payloadIndex) const -{ - if (payloadIndex >= nodeInfoTargetEntries()) - return 0; - return static_cast(payloadIndex + 1u); -} - -uint16_t TrafficManagementModule::decodeNodeInfoPayloadIndex(uint16_t tag) const -{ - if (tag == 0 || tag > nodeInfoTargetEntries()) - return UINT16_MAX; - return static_cast(tag - 1u); -} - -uint16_t TrafficManagementModule::getNodeInfoTag(uint16_t slot) const -{ -#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoIndex || slot >= nodeInfoIndexSlots()) - return 0; - - const uint32_t bitOffset = static_cast(slot) * nodeInfoTagBits(); - const uint16_t byteOffset = static_cast(bitOffset >> 3); - const uint8_t shift = static_cast(bitOffset & 7u); - uint32_t packed = 0; - - if (byteOffset < nodeInfoIndexMetadataBudgetBytes()) - packed |= static_cast(nodeInfoIndex[byteOffset]); - if (static_cast(byteOffset + 1u) < nodeInfoIndexMetadataBudgetBytes()) - packed |= static_cast(nodeInfoIndex[byteOffset + 1u]) << 8; - if (static_cast(byteOffset + 2u) < nodeInfoIndexMetadataBudgetBytes()) - packed |= static_cast(nodeInfoIndex[byteOffset + 2u]) << 16; - - return static_cast((packed >> shift) & nodeInfoTagMask()); -#else - (void)slot; - return 0; -#endif -} - -void TrafficManagementModule::setNodeInfoTag(uint16_t slot, uint16_t tag) -{ -#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoIndex || slot >= nodeInfoIndexSlots()) - return; - - const uint16_t normalizedTag = static_cast(tag & nodeInfoTagMask()); - const uint32_t bitOffset = static_cast(slot) * nodeInfoTagBits(); - const uint16_t byteOffset = static_cast(bitOffset >> 3); - const uint8_t shift = static_cast(bitOffset & 7u); - uint32_t packed = 0; - - if (byteOffset < nodeInfoIndexMetadataBudgetBytes()) - packed |= static_cast(nodeInfoIndex[byteOffset]); - if (static_cast(byteOffset + 1u) < nodeInfoIndexMetadataBudgetBytes()) - packed |= static_cast(nodeInfoIndex[byteOffset + 1u]) << 8; - if (static_cast(byteOffset + 2u) < nodeInfoIndexMetadataBudgetBytes()) - packed |= static_cast(nodeInfoIndex[byteOffset + 2u]) << 16; - - const uint32_t mask = static_cast(nodeInfoTagMask()) << shift; - packed = (packed & ~mask) | ((static_cast(normalizedTag) << shift) & mask); - - if (byteOffset < nodeInfoIndexMetadataBudgetBytes()) - nodeInfoIndex[byteOffset] = static_cast(packed & 0xFFu); - if (static_cast(byteOffset + 1u) < nodeInfoIndexMetadataBudgetBytes()) - nodeInfoIndex[byteOffset + 1u] = static_cast((packed >> 8) & 0xFFu); - if (static_cast(byteOffset + 2u) < nodeInfoIndexMetadataBudgetBytes()) - nodeInfoIndex[byteOffset + 2u] = static_cast((packed >> 16) & 0xFFu); -#else - (void)slot; - (void)tag; -#endif -} - -uint16_t TrafficManagementModule::findNodeInfoPayloadIndex(NodeNum node) const -{ -#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoPayload || !nodeInfoIndex || node == 0) - return UINT16_MAX; - - const uint16_t buckets[2] = {nodeInfoHash1(node), nodeInfoHash2(node)}; - - for (uint8_t b = 0; b < 2; b++) { - const uint16_t base = static_cast(buckets[b] * nodeInfoBucketSize()); - for (uint8_t slot = 0; slot < nodeInfoBucketSize(); slot++) { - uint16_t tag = getNodeInfoTag(static_cast(base + slot)); - if (tag == 0) - continue; - - uint16_t payloadIndex = decodeNodeInfoPayloadIndex(tag); - if (payloadIndex >= nodeInfoTargetEntries()) - continue; - - if (nodeInfoPayload[payloadIndex].node == node) - return payloadIndex; - } - } - - return UINT16_MAX; -#else - (void)node; - return UINT16_MAX; -#endif -} - -bool TrafficManagementModule::removeNodeInfoIndexEntry(NodeNum node, uint16_t payloadIndex) -{ -#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoIndex || node == 0 || payloadIndex >= nodeInfoTargetEntries()) - return false; - - const uint16_t payloadTag = encodeNodeInfoTag(payloadIndex); - if (payloadTag == 0) - return false; - const uint16_t buckets[2] = {nodeInfoHash1(node), nodeInfoHash2(node)}; - - for (uint8_t b = 0; b < 2; b++) { - const uint16_t base = static_cast(buckets[b] * nodeInfoBucketSize()); - for (uint8_t slot = 0; slot < nodeInfoBucketSize(); slot++) { - const uint16_t indexSlot = static_cast(base + slot); - if (getNodeInfoTag(indexSlot) == payloadTag) { - setNodeInfoTag(indexSlot, 0); - return true; - } - } - } - - return false; -#else - (void)node; - (void)payloadIndex; - return false; -#endif -} - -uint16_t TrafficManagementModule::allocateNodeInfoPayloadSlot() -{ -#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoPayload) - return UINT16_MAX; - - for (uint16_t tries = 0; tries < nodeInfoTargetEntries(); tries++) { - uint16_t idx = static_cast((nodeInfoAllocHint + tries) % nodeInfoTargetEntries()); - if (nodeInfoPayload[idx].node == 0) { - nodeInfoAllocHint = static_cast((idx + 1u) % nodeInfoTargetEntries()); - return idx; - } - } -#endif - return UINT16_MAX; -} - -uint16_t TrafficManagementModule::evictNodeInfoPayloadSlot() -{ -#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoPayload || !nodeInfoIndex) - return UINT16_MAX; - - for (uint16_t tries = 0; tries < nodeInfoTargetEntries(); tries++) { - uint16_t idx = static_cast(nodeInfoEvictCursor % nodeInfoTargetEntries()); - nodeInfoEvictCursor = static_cast((nodeInfoEvictCursor + 1u) % nodeInfoTargetEntries()); - - NodeNum oldNode = nodeInfoPayload[idx].node; - if (oldNode == 0) - continue; - - removeNodeInfoIndexEntry(oldNode, idx); // best effort; cache tolerates occasional stale miss - nodeInfoPayload[idx].node = 0; - return idx; - } -#endif - return UINT16_MAX; -} - -bool TrafficManagementModule::tryInsertNodeInfoEntryInBucket(uint16_t bucket, uint16_t tag) -{ -#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoIndex || !nodeInfoPayload || bucket >= nodeInfoBucketCount() || tag == 0) - return false; - - const uint16_t base = static_cast(bucket * nodeInfoBucketSize()); - for (uint8_t slot = 0; slot < nodeInfoBucketSize(); slot++) { - const uint16_t indexSlot = static_cast(base + slot); - const uint16_t existingTag = getNodeInfoTag(indexSlot); - if (existingTag == 0) { - setNodeInfoTag(indexSlot, tag); - return true; - } - - // Opportunistically reuse stale tags that point at empty/invalid payload slots. - const uint16_t payloadIndex = decodeNodeInfoPayloadIndex(existingTag); - if (payloadIndex >= nodeInfoTargetEntries() || nodeInfoPayload[payloadIndex].node == 0) { - setNodeInfoTag(indexSlot, tag); - return true; - } - } -#else - (void)bucket; - (void)tag; -#endif - return false; -} - +/** + * Find or create a NodeInfo payload entry (linear scan of the flat PSRAM + * array). One pass tracks the match, the first empty slot, and the LRU + * victim by lastObservedMs (wrap-safe age). NodeInfo traffic is low-rate, + * so the O(n) scan is negligible. + */ TrafficManagementModule::NodeInfoPayloadEntry *TrafficManagementModule::findOrCreateNodeInfoEntry(NodeNum node, bool *usedEmptySlot) { @@ -656,88 +411,40 @@ TrafficManagementModule::NodeInfoPayloadEntry *TrafficManagementModule::findOrCr *usedEmptySlot = false; #if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoPayload || !nodeInfoIndex || node == 0) + if (!nodeInfoPayload || node == 0) return nullptr; - uint16_t existing = findNodeInfoPayloadIndex(node); - if (existing < nodeInfoTargetEntries()) - return &nodeInfoPayload[existing]; + NodeInfoPayloadEntry *empty = nullptr; + NodeInfoPayloadEntry *lru = nullptr; + uint32_t lruAge = 0; + const uint32_t now = millis(); - const uint16_t beforeCount = countNodeInfoEntriesLocked(); - - uint16_t payloadIndex = allocateNodeInfoPayloadSlot(); - if (payloadIndex == UINT16_MAX) { - payloadIndex = evictNodeInfoPayloadSlot(); - if (payloadIndex == UINT16_MAX) - return nullptr; - } - - nodeInfoPayload[payloadIndex].node = node; - - // 4-way bucketed cuckoo insertion mirrors Cuckoo Filter practice from - // Fan et al. (CoNEXT 2014): high occupancy with short relocation chains. - uint16_t pending = encodeNodeInfoTag(payloadIndex); - uint16_t h1 = nodeInfoHash1(node); - uint16_t h2 = nodeInfoHash2(node); - - if (!tryInsertNodeInfoEntryInBucket(h1, pending) && !tryInsertNodeInfoEntryInBucket(h2, pending)) { - uint16_t currentBucket = h1; - for (uint8_t kicks = 0; kicks < kMaxCuckooKicks; kicks++) { - const uint16_t base = static_cast(currentBucket * nodeInfoBucketSize()); - const uint16_t kickSlot = static_cast((node + kicks) & (nodeInfoBucketSize() - 1u)); - const uint16_t pos = static_cast(base + kickSlot); - - uint16_t displaced = getNodeInfoTag(pos); - setNodeInfoTag(pos, pending); - pending = displaced; - - uint16_t displacedPayload = decodeNodeInfoPayloadIndex(pending); - if (displacedPayload >= nodeInfoTargetEntries()) { - pending = 0; - break; - } - - NodeNum displacedNode = nodeInfoPayload[displacedPayload].node; - if (displacedNode == 0) { - pending = 0; - break; - } - - uint16_t altH1 = nodeInfoHash1(displacedNode); - uint16_t altH2 = nodeInfoHash2(displacedNode); - uint16_t altBucket = (altH1 == currentBucket) ? altH2 : altH1; - - if (tryInsertNodeInfoEntryInBucket(altBucket, pending)) { - pending = 0; - break; - } - - currentBucket = altBucket; + for (uint16_t i = 0; i < nodeInfoTargetEntries(); i++) { + NodeInfoPayloadEntry &e = nodeInfoPayload[i]; + if (e.node == node) + return &e; + if (e.node == 0) { + if (!empty) + empty = &e; + continue; } - - if (pending != 0) { - uint16_t droppedPayload = decodeNodeInfoPayloadIndex(pending); - if (droppedPayload < nodeInfoTargetEntries()) - nodeInfoPayload[droppedPayload].node = 0; - TM_LOG_DEBUG("NodeInfo bucketed cuckoo overflow, dropped payload idx=%u", - static_cast(droppedPayload < nodeInfoTargetEntries() ? droppedPayload : UINT16_MAX)); + if (empty) + continue; // an empty slot beats any victim; stop scoring + const uint32_t age = now - e.lastObservedMs; // unsigned subtraction is wrap-safe + if (!lru || age > lruAge) { + lru = &e; + lruAge = age; } } - uint16_t finalIndex = findNodeInfoPayloadIndex(node); - if (finalIndex >= nodeInfoTargetEntries()) { - // New entry did not survive insertion chain. - if (payloadIndex < nodeInfoTargetEntries() && nodeInfoPayload[payloadIndex].node == node) - nodeInfoPayload[payloadIndex].node = 0; + NodeInfoPayloadEntry *slot = empty ? empty : lru; + if (!slot) return nullptr; - } - - if (usedEmptySlot) { - const uint16_t afterCount = countNodeInfoEntriesLocked(); - *usedEmptySlot = afterCount > beforeCount; - } - - return &nodeInfoPayload[finalIndex]; + memset(slot, 0, sizeof(NodeInfoPayloadEntry)); + slot->node = node; + if (usedEmptySlot) + *usedEmptySlot = (slot == empty); + return slot; #else (void)node; return nullptr; @@ -747,12 +454,12 @@ TrafficManagementModule::NodeInfoPayloadEntry *TrafficManagementModule::findOrCr uint16_t TrafficManagementModule::countNodeInfoEntriesLocked() const { #if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoIndex) + if (!nodeInfoPayload) return 0; uint16_t count = 0; - for (uint16_t i = 0; i < nodeInfoIndexSlots(); i++) { - if (getNodeInfoTag(i) != 0) + for (uint16_t i = 0; i < nodeInfoTargetEntries(); i++) { + if (nodeInfoPayload[i].node != 0) count++; } return count; @@ -764,7 +471,7 @@ uint16_t TrafficManagementModule::countNodeInfoEntriesLocked() const void TrafficManagementModule::cacheNodeInfoPacket(const meshtastic_MeshPacket &mp) { #if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoPayload || !nodeInfoIndex || mp.decoded.payload.size == 0) + if (!nodeInfoPayload || mp.decoded.payload.size == 0) return; meshtastic_User user = meshtastic_User_init_zero; @@ -797,16 +504,99 @@ void TrafficManagementModule::cacheNodeInfoPacket(const meshtastic_MeshPacket &m } if (usedEmptySlot) { - TM_LOG_INFO("NodeInfo PSRAM cache entries: %u/%u target (%u packed slots, %u-bit tags, %u-byte DRAM index)", - static_cast(cachedCount), static_cast(nodeInfoTargetEntries()), - static_cast(nodeInfoIndexSlots()), static_cast(nodeInfoTagBits()), - static_cast(nodeInfoIndexMetadataBudgetBytes())); + TM_LOG_INFO("NodeInfo PSRAM cache entries: %u/%u", static_cast(cachedCount), + static_cast(nodeInfoTargetEntries())); } #else (void)mp; #endif } +// ============================================================================= +// Next-Hop Overflow Cache +// ============================================================================= +// +// A routing hint store. The byte is the last byte of the NodeNum to use as next +// hop to reach `dest`. It is written ONLY from NextHopRouter's ACK-confirmed +// decision (a bidirectionally-verified relay) — never inferred one-way from +// relayed traffic. The TMM cache holds confirmed next-hops that have aged out of +// the hot NodeDB (NodeInfoLite), and NextHopRouter::getNextHop() consults it as a +// fallback after the hot store. + +void TrafficManagementModule::setNextHop(NodeNum dest, uint8_t nextHopByte) +{ +#if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 + if (!cache || dest == 0 || nextHopByte == 0) + return; + + concurrency::LockGuard guard(&cacheLock); + bool isNew = false; + UnifiedCacheEntry *entry = findOrCreateEntry(dest, &isNew); + if (entry) + entry->next_hop = nextHopByte; // last-write-wins; only confirmed bytes reach here +#else + (void)dest; + (void)nextHopByte; +#endif +} + +uint8_t TrafficManagementModule::getNextHopHint(NodeNum dest) +{ +#if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 + if (!cache || dest == 0) + return 0; + + concurrency::LockGuard guard(&cacheLock); + UnifiedCacheEntry *entry = findEntry(dest); + return entry ? entry->next_hop : 0; +#else + (void)dest; + return 0; +#endif +} + +void TrafficManagementModule::clearNextHop(NodeNum dest) +{ +#if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 + if (!cache || dest == 0) + return; + + concurrency::LockGuard guard(&cacheLock); + UnifiedCacheEntry *entry = findEntry(dest); + if (entry) + entry->next_hop = 0; // keep the entry (other stats), just drop the routing hint +#else + (void)dest; +#endif +} + +void TrafficManagementModule::preloadNextHopsFromNodeDB() +{ +#if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 + if (!cache || !nodeDB) + return; + + uint16_t seeded = 0; + concurrency::LockGuard guard(&cacheLock); + const size_t count = nodeDB->getNumMeshNodes(); + for (size_t i = 0; i < count; i++) { + const meshtastic_NodeInfoLite *node = nodeDB->getMeshNodeByIndex(i); + if (!node || node->num == 0 || node->next_hop == 0) + continue; + + bool isNew = false; + UnifiedCacheEntry *entry = findOrCreateEntry(node->num, &isNew); + // Don't clobber a freshly-learned confirmed hop with a (possibly stale) persisted one. + if (entry && entry->next_hop == 0) { + entry->next_hop = node->next_hop; + seeded++; + } + } + + TM_LOG_INFO("Preloaded %u next-hop hints from NodeDB", static_cast(seeded)); +#endif +} + // ============================================================================= // Epoch Management // ============================================================================= @@ -830,6 +620,43 @@ void TrafficManagementModule::resetEpoch(uint32_t nowMs) #endif } +/** + * Sliding-epoch rebase — preserve cached state past the 8-bit timestamp horizon. + * + * Instead of flushing the whole cache when offsets approach overflow, advance the + * epoch by a fixed slab and shift every live entry's relative timestamps back by + * the same wall-clock amount. A valid entry's window is only a handful of ticks + * wide (TTL auto-scales with resolution), so live entries comfortably survive; + * already-expired entries clamp to 0 and are reclaimed by the maintenance sweep in + * the same locked pass. Reconstructed absolute time is preserved (minus a sub-tick + * remainder), so in-flight TTL checks remain correct across the rebase. + * + * Caller must hold cacheLock. + */ +void TrafficManagementModule::rebaseEpoch(uint32_t nowMs) +{ +#if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 + (void)nowMs; + + // Slab stays well below the 200-tick reset threshold so a single rebase drops + // the offset back into range (~200 -> ~72 ticks) while live entries survive. + const uint32_t slabMs = 128UL * maxResolution() * 1000UL; + cacheEpochMs += slabMs; + + TM_LOG_DEBUG("Rebasing cache epoch by %lus", static_cast(slabMs / 1000UL)); + + for (uint16_t i = 0; i < cacheSize(); i++) { + if (cache[i].node == 0) + continue; + slideRelativeTime(cache[i].pos_time, slabMs, posTimeResolution); + slideRelativeTime(cache[i].rate_time, slabMs, rateTimeResolution); + slideRelativeTime(cache[i].unknown_time, slabMs, unknownTimeResolution); + } +#else + (void)nowMs; +#endif +} + // ============================================================================= // Position Hash (Compact Mode) // ============================================================================= @@ -1042,11 +869,12 @@ int32_t TrafficManagementModule::runOnce() #if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 const uint32_t nowMs = millis(); - // Check if epoch reset needed (~3.5 hours approaching 8-bit minute overflow) - if (needsEpochReset(nowMs)) { - concurrency::LockGuard guard(&cacheLock); - resetEpoch(nowMs); - return kMaintenanceIntervalMs; + // Warm-start the next-hop cache from persisted NodeInfoLite hints once nodeDB + // is populated. Done here (not in the constructor) so nodeDB has finished + // loading. Takes its own lock, so call before acquiring the sweep guard below. + if (!nextHopPreloaded) { + preloadNextHopsFromNodeDB(); + nextHopPreloaded = true; } // Calculate TTLs for cache expiration @@ -1065,6 +893,13 @@ int32_t TrafficManagementModule::runOnce() const uint32_t sweepStartMs = millis(); concurrency::LockGuard guard(&cacheLock); + + // Slide the epoch instead of flushing when offsets approach 8-bit overflow. + // Rebase preserves live entries; only already-expired ones clamp to 0 and are + // reclaimed by the sweep below in this same locked pass. + if (needsEpochReset(nowMs)) + rebaseEpoch(nowMs); + for (uint16_t i = 0; i < cacheSize(); i++) { if (cache[i].node == 0) continue; @@ -1104,6 +939,11 @@ int32_t TrafficManagementModule::runOnce() } } + // A confirmed next-hop hint has no TTL of its own and keeps the slot alive, + // so an aged-out routing hint outlives the dedup/rate/unknown state. + if (cache[i].next_hop != 0) + anyValid = true; + // If all data expired, free the slot entirely if (!anyValid) { memset(&cache[i], 0, sizeof(UnifiedCacheEntry)); @@ -1118,11 +958,9 @@ int32_t TrafficManagementModule::runOnce() static_cast(millis() - sweepStartMs)); #if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (nodeInfoPayload && nodeInfoIndex) { - TM_LOG_DEBUG("NodeInfo PSRAM cache: %u/%u target (%u packed slots, %u buckets, %u-bit tags, %u-byte index)", - static_cast(countNodeInfoEntriesLocked()), static_cast(nodeInfoTargetEntries()), - static_cast(nodeInfoIndexSlots()), static_cast(nodeInfoBucketCount()), - static_cast(nodeInfoTagBits()), static_cast(nodeInfoIndexMetadataBudgetBytes())); + if (nodeInfoPayload) { + TM_LOG_DEBUG("NodeInfo PSRAM cache: %u/%u", static_cast(countNodeInfoEntriesLocked()), + static_cast(nodeInfoTargetEntries())); } #endif @@ -1153,8 +991,11 @@ bool TrafficManagementModule::shouldDropPosition(const meshtastic_MeshPacket *p, const int32_t lat_truncated = truncateLatLon(pos->latitude_i, precision); const int32_t lon_truncated = truncateLatLon(pos->longitude_i, precision); const uint8_t fingerprint = computePositionFingerprint(lat_truncated, lon_truncated, precision); - const uint32_t minIntervalMs = secsToMs(Default::getConfiguredOrDefault( - moduleConfig.traffic_management.position_min_interval_secs, default_traffic_mgmt_position_min_interval_secs)); + // Drop gate uses the RAW configured interval: 0 means "dedup disabled" (the + // contract documented below). The 12h default is only for resolution/TTL + // sizing (constructor / runOnce), not for deciding whether to drop — feeding + // the default here would silently turn the 0-disables-dedup contract off. + const uint32_t minIntervalMs = secsToMs(moduleConfig.traffic_management.position_min_interval_secs); bool isNew = false; concurrency::LockGuard guard(&cacheLock); @@ -1218,7 +1059,7 @@ bool TrafficManagementModule::shouldRespondToNodeInfo(const meshtastic_MeshPacke // If the PSRAM cache exists but misses, we intentionally do not fall back // to the node-wide table. This keeps the PSRAM direct-reply path separate // from NodeInfoModule/NodeDB behavior when PSRAM is available. - if (nodeInfoPayload && nodeInfoIndex) { + if (nodeInfoPayload) { TM_LOG_DEBUG("NodeInfo PSRAM cache miss for node=0x%08x", p->to); return false; } @@ -1378,7 +1219,9 @@ bool TrafficManagementModule::shouldDropUnknown(const meshtastic_MeshPacket *p, entry->unknown_count = 0; } - // Increment counter (saturates at 255) + // Increment counter (saturates at 255). Same saturation handling as + // isRateLimited: without it, a clamped threshold of 255 can never fire. + const bool alreadySaturated = (entry->unknown_count == UINT8_MAX); saturatingIncrement(entry->unknown_count); // Check against threshold @@ -1386,7 +1229,7 @@ bool TrafficManagementModule::shouldDropUnknown(const meshtastic_MeshPacket *p, if (threshold > 255) threshold = 255; - bool drop = entry->unknown_count > threshold; + bool drop = entry->unknown_count > threshold || (alreadySaturated && threshold == 255); if (drop || entry->unknown_count == threshold) { TM_LOG_DEBUG("Unknown packets 0x%08x: count=%u threshold=%u -> %s", p->from, entry->unknown_count, threshold, drop ? "DROP" : "at-limit"); diff --git a/src/modules/TrafficManagementModule.h b/src/modules/TrafficManagementModule.h index fe3483a8e..b0e97d89c 100644 --- a/src/modules/TrafficManagementModule.h +++ b/src/modules/TrafficManagementModule.h @@ -20,9 +20,11 @@ * - Router hop preservation (maintain hop_limit for router-to-router traffic) * * Memory Optimization: - * Uses a unified cache with cuckoo hashing for O(1) lookups and 56% memory reduction - * compared to separate per-feature caches. Timestamps are stored as 8-bit relative - * offsets from a rolling epoch to further reduce memory footprint. + * Uses one flat unified cache (plain array, linear scan) shared by all + * per-node features instead of separate per-feature caches. Timestamps are + * stored as 8-bit relative offsets from a rolling epoch to further reduce + * memory footprint. LoRa packet rates are low enough that an O(n) scan of + * ~1000 11-byte entries is negligible next to packet processing. */ class TrafficManagementModule : public MeshModule, private concurrency::OSThread { @@ -38,6 +40,22 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread void resetStats(); void recordRouterHopPreserved(); + // Next-hop overflow cache (routing hint). + // setNextHop: store a confirmed last-byte next hop for `dest`. Called by + // NextHopRouter from its ACK-confirmed decision (see sniffReceived). The + // byte must come from a bidirectionally-verified relay, not one-way inference. + // getNextHopHint: return the cached next-hop byte for `dest`, 0 if unknown. + // clearNextHop: forget any cached next hop for `dest` (setNextHop refuses to store + // 0, so this is the way NextHopRouter decays a stale/failing overflow route). + void setNextHop(NodeNum dest, uint8_t nextHopByte); + uint8_t getNextHopHint(NodeNum dest); + void clearNextHop(NodeNum dest); + + // Warm-start the next-hop cache from persisted NodeInfoLite hints so confirmed + // hops survive later hot-store (NodeDB) eviction. Idempotent; runs once after + // nodeDB is populated (lazily on first maintenance pass). + void preloadNextHopsFromNodeDB(); + /** * Check if this packet should have its hops exhausted. * Called from perhapsRebroadcast() to force hop_limit = 0 regardless of @@ -55,14 +73,18 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread int32_t runOnce() override; // Protected so test shims can force epoch rollover behavior. void resetEpoch(uint32_t nowMs); + // Sliding-epoch rebase: advance the epoch and shift live entries back by the + // same wall-clock amount instead of flushing, so cached state survives past the + // ~19h horizon. Caller must hold cacheLock. + void rebaseEpoch(uint32_t nowMs); private: // ========================================================================= - // Unified Cache Entry (10 bytes) - Same for ALL platforms + // Unified Cache Entry (11 bytes) - Same for ALL platforms // ========================================================================= // // A single compact structure used across ESP32, NRF52, and all other platforms. - // Memory: 10 bytes × 2048 entries = 20KB + // Memory: 11 bytes × TRAFFIC_MANAGEMENT_CACHE_SIZE entries (default 1000 = 11KB) // // Position Fingerprinting: // Instead of storing full coordinates (8 bytes) or a computed hash, @@ -90,6 +112,16 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread // [7] pos_time - Position timestamp (1 byte, adaptive resolution) // [8] rate_time - Rate window start (1 byte, adaptive resolution) // [9] unknown_time - Unknown tracking start (1 byte, adaptive resolution) + // [10] next_hop - Last-byte relay to reach `node` (1 byte, 0 = none) + // + // next_hop semantics: + // A routing hint: the last byte of the NodeNum to use as next hop to reach + // `node`. Written ONLY from NextHopRouter's ACK-confirmed decision (a + // bidirectionally-verified relay), never inferred one-way from relayed + // traffic. The TMM cache acts as an overflow store for confirmed next-hops + // that have aged out of the hot NodeDB (NodeInfoLite). Unlike the other + // fields it has no TTL of its own — it keeps its slot alive (see runOnce) + // and is refreshed only on the next confirmed exchange. // struct __attribute__((packed)) UnifiedCacheEntry { NodeNum node; // 4 bytes - Node identifier (0 = empty slot) @@ -99,66 +131,27 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread uint8_t pos_time; // 1 byte - Position timestamp (adaptive resolution) uint8_t rate_time; // 1 byte - Rate window start (adaptive resolution) uint8_t unknown_time; // 1 byte - Unknown tracking start (adaptive resolution) + uint8_t next_hop; // 1 byte - Last-byte relay to reach `node` (0 = none). See note below. }; - static_assert(sizeof(UnifiedCacheEntry) == 10, "UnifiedCacheEntry should be 10 bytes"); + static_assert(sizeof(UnifiedCacheEntry) == 11, "UnifiedCacheEntry should be 11 bytes"); // ========================================================================= - // Cuckoo Hash Table Implementation + // Flat unified cache // ========================================================================= // - // Cuckoo hashing provides O(1) worst-case lookup time using two hash functions. - // Each key can be in one of two possible locations (h1 or h2). On collision, - // the existing entry is "kicked" to its alternate location. + // Plain array, linear scan (same idiom as WarmNodeStore). A lookup walks at + // most cacheSize() × 11 B — microseconds at LoRa packet rates, not worth a + // hash table. Insertion on a full cache evicts the stalest entry, + // preferring entries without a next_hop hint (those are the long-tail + // routing state this cache exists to keep). // - // Benefits over linear scan: - // - O(1) lookup vs O(n) - critical at packet processing rates - // - O(1) insertion (amortized) with simple eviction on cycles - // - ~95% load factor achievable - // - // Cache size rounds to power-of-2 for fast modulo via bitmask. - // TRAFFIC_MANAGEMENT_CACHE_SIZE=2000 → cacheSize()=2048 - // - static constexpr uint16_t cacheSize(); - static constexpr uint16_t cacheMask(); + static constexpr uint16_t cacheSize() { return TRAFFIC_MANAGEMENT_CACHE_SIZE; } - // Hash functions for cuckoo hashing - inline uint16_t cuckooHash1(NodeNum node) const { return node & cacheMask(); } - inline uint16_t cuckooHash2(NodeNum node) const { return ((node * 2654435769u) >> (32 - cuckooHashBits())) & cacheMask(); } - static constexpr uint8_t cuckooHashBits(); - - // NodeInfo cache configuration (PSRAM path): - // - Payload lives in PSRAM - // - DRAM keeps packed 12-bit tags with 4-way bucketed cuckoo hashing - // (Fan et al., CoNEXT 2014). Tag value 0 is reserved as "empty". - static constexpr uint16_t kNodeInfoIndexMetadataBudgetBytes = 3072; // 3KB DRAM tag store - static constexpr uint8_t kNodeInfoTargetOccupancyPercent = 95; - static constexpr uint8_t kNodeInfoBucketSize = 4; - static constexpr uint8_t kNodeInfoTagBits = 12; - static constexpr uint16_t kNodeInfoTagMask = static_cast((1u << kNodeInfoTagBits) - 1u); - static constexpr uint16_t kNodeInfoIndexSlotsRaw = - static_cast((kNodeInfoIndexMetadataBudgetBytes * 8u) / kNodeInfoTagBits); - static constexpr uint16_t kNodeInfoIndexSlots = - static_cast(kNodeInfoIndexSlotsRaw - (kNodeInfoIndexSlotsRaw % kNodeInfoBucketSize)); - static constexpr uint16_t kNodeInfoTargetEntries = - static_cast((kNodeInfoIndexSlots * kNodeInfoTargetOccupancyPercent) / 100u); - static_assert((kNodeInfoIndexSlots % kNodeInfoBucketSize) == 0, "NodeInfo slot count must align to bucket size"); - static_assert(kNodeInfoTargetEntries < (1u << kNodeInfoTagBits), "NodeInfo tag bits must encode payload index"); - - static constexpr uint16_t nodeInfoTargetEntries(); - static constexpr uint16_t nodeInfoIndexMetadataBudgetBytes(); - static constexpr uint8_t nodeInfoTargetOccupancyPercent(); - static constexpr uint8_t nodeInfoBucketSize(); - static constexpr uint8_t nodeInfoTagBits(); - static constexpr uint16_t nodeInfoTagMask(); - static constexpr uint16_t nodeInfoIndexSlots(); - static constexpr uint16_t nodeInfoBucketCount(); - static constexpr uint16_t nodeInfoBucketMask(); - static constexpr uint8_t nodeInfoBucketHashBits(); - inline uint16_t nodeInfoHash1(NodeNum node) const { return node & nodeInfoBucketMask(); } - inline uint16_t nodeInfoHash2(NodeNum node) const - { - return ((node * 2246822519u) >> (32 - nodeInfoBucketHashBits())) & nodeInfoBucketMask(); - } + // NodeInfo cache configuration (PSRAM path): a flat PSRAM array of payload + // entries, linear scan keyed by `node`, LRU eviction by lastObservedMs. + // NodeInfo traffic is low-rate, so a full scan per lookup/insert is fine. + static constexpr uint16_t kNodeInfoCacheEntries = 2000; + static constexpr uint16_t nodeInfoTargetEntries() { return kNodeInfoCacheEntries; } // ========================================================================= // Adaptive Timestamp Resolution @@ -192,18 +185,28 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread return static_cast(res); } - // Convert to/from 8-bit relative timestamps with given resolution + // Convert to/from 8-bit relative timestamps with given resolution. + // + // All stored timestamps carry a uniform +1 "presence" offset: a value of 0 is + // reserved for "no timestamp recorded" (which is also the zero-initialized + // state), and stored values 1..255 encode raw ticks 0..254. This keeps the + // 0-means-empty sentinel consistent with memset/calloc zeroing across every + // sub-store, so the maintenance sweep's `_time != 0` presence checks are + // unambiguous (a timestamp recorded in the first tick after the epoch is no + // longer mistaken for an empty slot). The offset is applied here and removed + // on read, so it cancels out in all window math. uint8_t toRelativeTime(uint32_t nowMs, uint16_t resolutionSecs) const { uint32_t ticks = (nowMs - cacheEpochMs) / (resolutionSecs * 1000UL); - return (ticks > UINT8_MAX) ? UINT8_MAX : static_cast(ticks); + return (ticks >= UINT8_MAX) ? UINT8_MAX : static_cast(ticks + 1); } uint32_t fromRelativeTime(uint8_t ticks, uint16_t resolutionSecs) const { - return cacheEpochMs + (static_cast(ticks) * resolutionSecs * 1000UL); + return (ticks == 0) ? cacheEpochMs : cacheEpochMs + (static_cast(ticks - 1) * resolutionSecs * 1000UL); } - // Convenience wrappers for each timestamp type + // Convenience wrappers for each timestamp type (the +1 presence offset lives + // in the shared converters above, so these are plain pass-throughs). uint8_t toRelativePosTime(uint32_t nowMs) const { return toRelativeTime(nowMs, posTimeResolution); } uint32_t fromRelativePosTime(uint8_t t) const { return fromRelativeTime(t, posTimeResolution); } @@ -213,17 +216,20 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread uint8_t toRelativeUnknownTime(uint32_t nowMs) const { return toRelativeTime(nowMs, unknownTimeResolution); } uint32_t fromRelativeUnknownTime(uint8_t t) const { return fromRelativeTime(t, unknownTimeResolution); } - // Epoch reset when any timestamp approaches overflow - // With max resolution of 339 sec, 200 ticks = ~19 hours (safe margin for 24h max) - bool needsEpochReset(uint32_t nowMs) const + // Coarsest of the per-feature resolutions (seconds per tick). + uint16_t maxResolution() const { uint16_t maxRes = posTimeResolution; if (rateTimeResolution > maxRes) maxRes = rateTimeResolution; if (unknownTimeResolution > maxRes) maxRes = unknownTimeResolution; - return (nowMs - cacheEpochMs) > (200UL * maxRes * 1000UL); + return maxRes; } + + // True when relative offsets approach 8-bit overflow. + // With max resolution of 339 sec, 200 ticks = ~19 hours (safe margin for 24h max). + bool needsEpochReset(uint32_t nowMs) const { return (nowMs - cacheEpochMs) > (200UL * maxResolution() * 1000UL); } // ========================================================================= // Position Fingerprint // ========================================================================= @@ -246,7 +252,7 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread // ========================================================================= mutable concurrency::Lock cacheLock; // Protects all cache access - UnifiedCacheEntry *cache = nullptr; // Cuckoo hash table (unified for all platforms) + UnifiedCacheEntry *cache = nullptr; // Flat unified cache (linear scan; all platforms) bool cacheFromPsram = false; // Tracks allocator for correct deallocation struct NodeInfoPayloadEntry { @@ -278,11 +284,8 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread uint8_t decodedBitfield; }; - NodeInfoPayloadEntry *nodeInfoPayload = nullptr; // NodeInfo payloads in PSRAM + NodeInfoPayloadEntry *nodeInfoPayload = nullptr; // NodeInfo payloads in PSRAM (flat array, linear scan) bool nodeInfoPayloadFromPsram = false; // Tracks allocator for correct deallocation - uint8_t *nodeInfoIndex = nullptr; // Packed 12-bit NodeInfo tags in DRAM - uint16_t nodeInfoAllocHint = 0; - uint16_t nodeInfoEvictCursor = 0; meshtastic_TrafficManagementStats stats; @@ -293,29 +296,22 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread NodeNum exhaustRequestedFrom = 0; PacketId exhaustRequestedId = 0; + // One-shot guard: warm-start next-hop cache from NodeDB on first maintenance pass. + bool nextHopPreloaded = false; + // ========================================================================= // Cache Operations // ========================================================================= - // Find or create entry for node using cuckoo hashing - // Returns nullptr if cache is full and eviction fails + // Find or create entry for node (linear scan; stalest-first eviction when full) UnifiedCacheEntry *findOrCreateEntry(NodeNum node, bool *isNew); // Find existing entry (no creation) UnifiedCacheEntry *findEntry(NodeNum node); - // NodeInfo cache operations (bucketed cuckoo index + PSRAM payloads) + // NodeInfo cache operations (flat PSRAM payload array, linear scan) const NodeInfoPayloadEntry *findNodeInfoEntry(NodeNum node) const; NodeInfoPayloadEntry *findOrCreateNodeInfoEntry(NodeNum node, bool *usedEmptySlot); - uint16_t findNodeInfoPayloadIndex(NodeNum node) const; - bool removeNodeInfoIndexEntry(NodeNum node, uint16_t payloadIndex); - uint16_t allocateNodeInfoPayloadSlot(); - uint16_t evictNodeInfoPayloadSlot(); - bool tryInsertNodeInfoEntryInBucket(uint16_t bucket, uint16_t tag); - uint16_t encodeNodeInfoTag(uint16_t payloadIndex) const; - uint16_t decodeNodeInfoPayloadIndex(uint16_t tag) const; - uint16_t getNodeInfoTag(uint16_t slot) const; - void setNodeInfoTag(uint16_t slot, uint16_t tag); uint16_t countNodeInfoEntriesLocked() const; void cacheNodeInfoPacket(const meshtastic_MeshPacket &mp); @@ -333,101 +329,7 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread void incrementStat(uint32_t *field); }; -// ========================================================================= -// Compile-time Cache Size Calculations -// ========================================================================= -// -// Round TRAFFIC_MANAGEMENT_CACHE_SIZE up to next power of 2 for efficient -// cuckoo hash indexing (allows bitmask instead of modulo). -// -// These use C++11-compatible constexpr (single return statement). -// - -namespace detail -{ -// Helper: round up to next power of 2 using bit manipulation -constexpr uint16_t nextPow2(uint16_t n) -{ - return n == 0 ? 0 : (((n - 1) | ((n - 1) >> 1) | ((n - 1) >> 2) | ((n - 1) >> 4) | ((n - 1) >> 8)) + 1); -} - -// Helper: floor(log2(n)) for n >= 0, C++11-compatible constexpr. -constexpr uint8_t log2Floor(uint16_t n) -{ - return n <= 1 ? 0 : static_cast(1 + log2Floor(static_cast(n >> 1))); -} - -// Helper: ceil(log2(n)) for n >= 1, C++11-compatible constexpr. -constexpr uint8_t log2Ceil(uint16_t n) -{ - return n <= 1 ? 0 : static_cast(1 + log2Floor(static_cast(n - 1))); -} -} // namespace detail - -constexpr uint16_t TrafficManagementModule::cacheSize() -{ - return detail::nextPow2(TRAFFIC_MANAGEMENT_CACHE_SIZE); -} - -constexpr uint16_t TrafficManagementModule::cacheMask() -{ - return cacheSize() > 0 ? cacheSize() - 1 : 0; -} - -constexpr uint8_t TrafficManagementModule::cuckooHashBits() -{ - return detail::log2Floor(cacheSize()); -} - -constexpr uint16_t TrafficManagementModule::nodeInfoTargetEntries() -{ - return kNodeInfoTargetEntries; -} - -constexpr uint16_t TrafficManagementModule::nodeInfoIndexMetadataBudgetBytes() -{ - return kNodeInfoIndexMetadataBudgetBytes; -} - -constexpr uint8_t TrafficManagementModule::nodeInfoTargetOccupancyPercent() -{ - return kNodeInfoTargetOccupancyPercent; -} - -constexpr uint8_t TrafficManagementModule::nodeInfoBucketSize() -{ - return kNodeInfoBucketSize; -} - -constexpr uint8_t TrafficManagementModule::nodeInfoTagBits() -{ - return kNodeInfoTagBits; -} - -constexpr uint16_t TrafficManagementModule::nodeInfoTagMask() -{ - return kNodeInfoTagMask; -} - -constexpr uint16_t TrafficManagementModule::nodeInfoIndexSlots() -{ - return kNodeInfoIndexSlots; -} - -constexpr uint16_t TrafficManagementModule::nodeInfoBucketCount() -{ - return static_cast(nodeInfoIndexSlots() / nodeInfoBucketSize()); -} - -constexpr uint16_t TrafficManagementModule::nodeInfoBucketMask() -{ - return nodeInfoBucketCount() > 0 ? nodeInfoBucketCount() - 1 : 0; -} - -constexpr uint8_t TrafficManagementModule::nodeInfoBucketHashBits() -{ - return detail::log2Floor(nodeInfoBucketCount()); -} +static_assert(TRAFFIC_MANAGEMENT_CACHE_SIZE <= UINT16_MAX, "cacheSize() returns uint16_t"); extern TrafficManagementModule *trafficManagementModule; diff --git a/test/test_nexthop_routing/test_main.cpp b/test/test_nexthop_routing/test_main.cpp new file mode 100644 index 000000000..e766cbdee --- /dev/null +++ b/test/test_nexthop_routing/test_main.cpp @@ -0,0 +1,465 @@ +// Unit tests for NextHop direct-message reliability mitigations (see docs/nexthop-routing-reliability.md): +// M1 - NodeDB::resolveLastByte / resolveUniqueLastByte (ambiguity-aware last-byte resolution) +// M2 - NextHopRouter::getNextHop strict-neighbor gate + Router::shouldDecrementHopLimit favorite check +// M3 - NextHopRouter route-health freshness / failure decay +// +// Time handling: the route-health helpers take `now` as a parameter so the 30-minute TTL logic is +// pure and testable without a clock mock. getNextHop()/sinceLastSeen() use the real native clock; +// we back-date timestamps relative to it, and the unsigned-subtraction age math is rollover-safe. + +#include "MeshTypes.h" // before TestUtil.h: provides NodeNum etc. +#include "TestUtil.h" +#include + +#include "configuration.h" +#include "gps/RTC.h" +#include "mesh/NextHopRouter.h" +#include "mesh/NodeDB.h" +#include +#include +#include + +#define MSG_BUF_LEN 200 +#define TEST_MSG_FMT(fmt, ...) \ + do { \ + char _buf[MSG_BUF_LEN]; \ + snprintf(_buf, sizeof(_buf), fmt, __VA_ARGS__); \ + TEST_MESSAGE(_buf); \ + } while (0) + +static constexpr NodeNum kLocalNode = 0x11111111; // last byte 0x11 + +// --------------------------------------------------------------------------- +// MockNodeDB — inject nodes with controlled last byte, hop distance, age, role, favorite flag. +// --------------------------------------------------------------------------- +class MockNodeDB : public NodeDB +{ + public: + void clearTestNodes() + { + testNodes.clear(); + meshNodes = &testNodes; + numMeshNodes = 0; + } + + // ageSecs is how long ago we last heard the node; getTime() returns a large Unix timestamp on + // native, so getTime()-ageSecs does not underflow for the ranges used here. + void addNode(NodeNum num, uint8_t hopsAway, bool hasHops, uint32_t ageSecs, + meshtastic_Config_DeviceConfig_Role role = meshtastic_Config_DeviceConfig_Role_CLIENT, bool favorite = false, + bool ignored = false, uint8_t nextHop = NO_NEXT_HOP_PREFERENCE) + { + meshtastic_NodeInfoLite node = meshtastic_NodeInfoLite_init_zero; + node.num = num; + node.has_hops_away = hasHops; + node.hops_away = hopsAway; + node.role = role; + node.next_hop = nextHop; + node.last_heard = getTime() - ageSecs; + nodeInfoLiteSetBit(&node, NODEINFO_BITFIELD_IS_FAVORITE_MASK, favorite); + nodeInfoLiteSetBit(&node, NODEINFO_BITFIELD_IS_IGNORED_MASK, ignored); + nodeInfoLiteSetBit(&node, NODEINFO_BITFIELD_HAS_USER_MASK, true); + testNodes.push_back(node); + meshNodes = &testNodes; + numMeshNodes = testNodes.size(); + } + + std::vector testNodes; +}; + +// --------------------------------------------------------------------------- +// Test shim — expose getNextHop and the route-health helpers; reset health between tests. +// Nulls cryptLock so the Router base can be (re)constructed (same pattern as test_mqtt MockRouter). +// --------------------------------------------------------------------------- +class NextHopRouterTestShim : public NextHopRouter +{ + public: + NextHopRouterTestShim() : NextHopRouter() + { + delete cryptLock; + cryptLock = nullptr; + } + + using NextHopRouter::clearRouteHealth; + using NextHopRouter::findRouteHealth; + using NextHopRouter::getNextHop; + using NextHopRouter::getOrAllocRouteHealth; + using NextHopRouter::isRouteStale; + using NextHopRouter::noteRouteFailure; + using NextHopRouter::noteRouteLearned; + using NextHopRouter::noteRouteSuccess; + using Router::shouldDecrementHopLimit; // protected in Router + + void resetRouteHealthForTest() + { + for (auto &h : routeHealth) + h = RouteHealth{}; + } +}; + +static MockNodeDB *mockNodeDB = nullptr; +static NextHopRouterTestShim *shim = nullptr; + +static constexpr uint32_t TTL = NextHopRouter::ROUTE_TTL_MSEC; +static constexpr uint8_t THRESH = NextHopRouter::ROUTE_FAILURE_THRESHOLD; +static constexpr uint8_t HEALTH_MAX = NextHopRouter::ROUTE_HEALTH_MAX; + +// Helper: a decoded packet whose hops-away is `hopsAway`, relayed by last byte `relay`. +static meshtastic_MeshPacket makeRelayedPacket(uint8_t relay, uint8_t hopsAway) +{ + meshtastic_MeshPacket p = meshtastic_MeshPacket_init_zero; + p.which_payload_variant = meshtastic_MeshPacket_decoded_tag; + p.relay_node = relay; + p.hop_start = 4; + p.hop_limit = 4 - hopsAway; // getHopsAway() == hop_start - hop_limit + return p; +} + +void setUp(void) +{ + myNodeInfo.my_node_num = kLocalNode; + config.device.role = meshtastic_Config_DeviceConfig_Role_CLIENT; + mockNodeDB->clearTestNodes(); + shim->resetRouteHealthForTest(); +} + +void tearDown(void) {} + +// =========================================================================== +// Group 1 — resolveLastByte (M1) +// =========================================================================== + +void test_resolve_none_when_empty(void) +{ + ResolvedNode r = mockNodeDB->resolveLastByte(0xAB, true); + TEST_ASSERT_EQUAL(LastByteResolution::None, r.status); +} + +void test_resolve_zero_byte_is_none(void) +{ + // 0 is the NO_RELAY_NODE / NO_NEXT_HOP_PREFERENCE sentinel — never resolves. + mockNodeDB->addNode(0x22222200, 0, true, 60); // last byte maps to 0xFF, not 0 + TEST_ASSERT_EQUAL(LastByteResolution::None, mockNodeDB->resolveLastByte(0x00, true).status); + TEST_ASSERT_EQUAL(LastByteResolution::None, mockNodeDB->resolveLastByte(0x00, false).status); +} + +void test_resolve_unique_neighbor(void) +{ + mockNodeDB->addNode(0x000005AB, 0, true, 60); // direct, fresh, last byte 0xAB + ResolvedNode r = mockNodeDB->resolveLastByte(0xAB, true); + TEST_ASSERT_EQUAL(LastByteResolution::Unique, r.status); + TEST_ASSERT_EQUAL_HEX32(0x000005AB, r.num); +} + +void test_resolve_collision_is_ambiguous(void) +{ + // Birthday collision: two fresh direct neighbors share last byte 0xAB. + mockNodeDB->addNode(0x000005AB, 0, true, 60); + mockNodeDB->addNode(0x000006AB, 0, true, 60); + ResolvedNode r = mockNodeDB->resolveLastByte(0xAB, true); + TEST_ASSERT_EQUAL(LastByteResolution::Ambiguous, r.status); + TEST_ASSERT_EQUAL_HEX32(0, r.num); // never silently picks one +} + +void test_resolve_strict_excludes_stale(void) +{ + mockNodeDB->addNode(0x000005AB, 0, true, 60); // fresh + mockNodeDB->addNode(0x000006AB, 0, true, NEXTHOP_NEIGHBOR_FRESH_SECS + 100); // stale + ResolvedNode r = mockNodeDB->resolveLastByte(0xAB, true); + TEST_ASSERT_EQUAL(LastByteResolution::Unique, r.status); + TEST_ASSERT_EQUAL_HEX32(0x000005AB, r.num); +} + +void test_resolve_strict_excludes_far(void) +{ + mockNodeDB->addNode(0x000005AB, 0, true, 60); // direct neighbor + mockNodeDB->addNode(0x000006AB, 2, true, 60); // 2 hops away -> not a direct neighbor + ResolvedNode r = mockNodeDB->resolveLastByte(0xAB, true); + TEST_ASSERT_EQUAL(LastByteResolution::Unique, r.status); + TEST_ASSERT_EQUAL_HEX32(0x000005AB, r.num); +} + +void test_resolve_lenient_includes_favorite_router(void) +{ + // Unknown hop distance, but a favorite ROUTER: lenient gate accepts, strict gate does not. + mockNodeDB->addNode(0x000007AB, 0, false, 60, meshtastic_Config_DeviceConfig_Role_ROUTER, /*favorite=*/true); + TEST_ASSERT_EQUAL(LastByteResolution::Unique, mockNodeDB->resolveLastByte(0xAB, false).status); + TEST_ASSERT_EQUAL(LastByteResolution::None, mockNodeDB->resolveLastByte(0xAB, true).status); +} + +void test_resolve_lenient_collision_favorite_plus_neighbor(void) +{ + mockNodeDB->addNode(0x000007AB, 0, false, 60, meshtastic_Config_DeviceConfig_Role_ROUTER, /*favorite=*/true); + mockNodeDB->addNode(0x000005AB, 0, true, 60); // direct neighbor, same byte + TEST_ASSERT_EQUAL(LastByteResolution::Ambiguous, mockNodeDB->resolveLastByte(0xAB, false).status); +} + +void test_resolve_skips_self(void) +{ + // A node equal to us (last byte 0x11) must never resolve to ourselves. + mockNodeDB->addNode(kLocalNode, 0, true, 60); + TEST_ASSERT_EQUAL(LastByteResolution::None, mockNodeDB->resolveLastByte(0x11, true).status); +} + +void test_resolve_skips_ignored(void) +{ + mockNodeDB->addNode(0x000005AB, 0, true, 60, meshtastic_Config_DeviceConfig_Role_CLIENT, false, /*ignored=*/true); + TEST_ASSERT_EQUAL(LastByteResolution::None, mockNodeDB->resolveLastByte(0xAB, true).status); +} + +void test_resolve_0x00_maps_to_0xFF_and_collides(void) +{ + // getLastByteOfNodeNum() maps ...00 -> 0xFF, so a ...00 node and a ...FF node collide on 0xFF. + TEST_ASSERT_EQUAL_HEX8(0xFF, mockNodeDB->getLastByteOfNodeNum(0x11111100)); + mockNodeDB->addNode(0x11111100, 0, true, 60); // last byte 0xFF (remapped) + TEST_ASSERT_EQUAL(LastByteResolution::Unique, mockNodeDB->resolveLastByte(0xFF, true).status); + mockNodeDB->addNode(0x222222FF, 0, true, 60); // genuine ...FF + TEST_ASSERT_EQUAL(LastByteResolution::Ambiguous, mockNodeDB->resolveLastByte(0xFF, true).status); +} + +void test_resolve_unique_helper(void) +{ + mockNodeDB->addNode(0x000005AB, 0, true, 60); + NodeNum out = 0; + TEST_ASSERT_TRUE(mockNodeDB->resolveUniqueLastByte(0xAB, true, &out)); + TEST_ASSERT_EQUAL_HEX32(0x000005AB, out); + mockNodeDB->addNode(0x000006AB, 0, true, 60); // create collision + TEST_ASSERT_FALSE(mockNodeDB->resolveUniqueLastByte(0xAB, true)); +} + +// =========================================================================== +// Group 2 — getNextHop (M2 send-path gate + M3 decay) +// =========================================================================== + +static constexpr NodeNum DEST = 0x000000B0; // DM destination (last byte 0xB0, distinct from 0xAB) + +void test_getnexthop_unique_returns_byte(void) +{ + mockNodeDB->addNode(DEST, 2, true, 60, meshtastic_Config_DeviceConfig_Role_CLIENT, false, false, /*nextHop=*/0xAB); + mockNodeDB->addNode(0x000005AB, 0, true, 60); // unique fresh neighbor with byte 0xAB + auto nh = shim->getNextHop(DEST, /*relay=*/0x11); + TEST_ASSERT_TRUE(nh.has_value()); + TEST_ASSERT_EQUAL_HEX8(0xAB, nh.value()); +} + +void test_getnexthop_ambiguous_floods(void) +{ + mockNodeDB->addNode(DEST, 2, true, 60, meshtastic_Config_DeviceConfig_Role_CLIENT, false, false, /*nextHop=*/0xAB); + mockNodeDB->addNode(0x000005AB, 0, true, 60); + mockNodeDB->addNode(0x000006AB, 0, true, 60); // collision -> ambiguous + TEST_ASSERT_FALSE(shim->getNextHop(DEST, 0x11).has_value()); +} + +void test_getnexthop_vanished_neighbor_floods(void) +{ + mockNodeDB->addNode(DEST, 2, true, 60, meshtastic_Config_DeviceConfig_Role_CLIENT, false, false, /*nextHop=*/0xAB); + // The only 0xAB node is stale -> strict gate yields None -> flood. + mockNodeDB->addNode(0x000005AB, 0, true, NEXTHOP_NEIGHBOR_FRESH_SECS + 100); + TEST_ASSERT_FALSE(shim->getNextHop(DEST, 0x11).has_value()); +} + +void test_getnexthop_split_horizon_floods(void) +{ + mockNodeDB->addNode(DEST, 2, true, 60, meshtastic_Config_DeviceConfig_Role_CLIENT, false, false, /*nextHop=*/0xAB); + mockNodeDB->addNode(0x000005AB, 0, true, 60); + // relay_node == stored next_hop -> don't send it back the way it came. + TEST_ASSERT_FALSE(shim->getNextHop(DEST, /*relay=*/0xAB).has_value()); +} + +void test_getnexthop_broadcast_is_nullopt(void) +{ + TEST_ASSERT_FALSE(shim->getNextHop(NODENUM_BROADCAST, 0x11).has_value()); +} + +void test_getnexthop_decays_stale_route(void) +{ + mockNodeDB->addNode(DEST, 2, true, 60, meshtastic_Config_DeviceConfig_Role_CLIENT, false, false, /*nextHop=*/0xAB); + mockNodeDB->addNode(0x000005AB, 0, true, 60); // a valid unique neighbor exists... + // ...but the health record is older than the TTL, so the route should decay to flooding. + shim->noteRouteLearned(DEST, 0xAB, millis() - (TTL + 5000)); + TEST_ASSERT_FALSE(shim->getNextHop(DEST, 0x11).has_value()); + // Decay also clears the persisted next_hop and the RAM health record. + TEST_ASSERT_EQUAL_HEX8(NO_NEXT_HOP_PREFERENCE, mockNodeDB->getMeshNode(DEST)->next_hop); + TEST_ASSERT_NULL(shim->findRouteHealth(DEST)); +} + +// =========================================================================== +// Group 3 — route-health helpers (M3) +// =========================================================================== + +void test_health_fresh_not_stale(void) +{ + shim->noteRouteLearned(DEST, 0xAB, 1000); + RouteHealth *h = shim->findRouteHealth(DEST); + TEST_ASSERT_NOT_NULL(h); + TEST_ASSERT_FALSE(shim->isRouteStale(*h, 1000 + TTL - 1)); +} + +void test_health_ttl_expiry(void) +{ + shim->noteRouteLearned(DEST, 0xAB, 1000); + RouteHealth *h = shim->findRouteHealth(DEST); + TEST_ASSERT_TRUE(shim->isRouteStale(*h, 1000 + TTL)); // boundary is inclusive (>=) +} + +void test_health_ttl_rollover_safe(void) +{ + const uint32_t learnAt = 0xFFFFFFFFu - 1000; // learned just before the millis() rollover + shim->noteRouteLearned(DEST, 0xAB, learnAt); + RouteHealth *h = shim->findRouteHealth(DEST); + // 1500 ms later (wrapped to now=500): unsigned subtraction yields ~1500 ms, not "stale". + TEST_ASSERT_FALSE(shim->isRouteStale(*h, 500)); +} + +void test_health_failure_threshold(void) +{ + shim->noteRouteLearned(DEST, 0xAB, 1000); + for (uint8_t i = 1; i < THRESH; i++) + shim->noteRouteFailure(DEST); + TEST_ASSERT_FALSE(shim->isRouteStale(*shim->findRouteHealth(DEST), 1000)); // THRESH-1 failures: ok + shim->noteRouteFailure(DEST); + TEST_ASSERT_TRUE(shim->isRouteStale(*shim->findRouteHealth(DEST), 1000)); // THRESH failures: stale +} + +void test_health_success_resets_failures(void) +{ + shim->noteRouteLearned(DEST, 0xAB, 1000); + shim->noteRouteFailure(DEST); + shim->noteRouteFailure(DEST); + shim->noteRouteSuccess(DEST, 2000); + RouteHealth *h = shim->findRouteHealth(DEST); + TEST_ASSERT_EQUAL_UINT8(0, h->consecutiveFailures); + TEST_ASSERT_FALSE(shim->isRouteStale(*h, 2000)); +} + +void test_health_relearn_same_hop_keeps_failures(void) +{ + // Anti-flap: an asymmetric reverse path re-teaching the same dead hop must not reset failures. + shim->noteRouteLearned(DEST, 0xAB, 1000); + shim->noteRouteFailure(DEST); + shim->noteRouteFailure(DEST); + shim->noteRouteLearned(DEST, 0xAB, 2000); // same hop + TEST_ASSERT_EQUAL_UINT8(2, shim->findRouteHealth(DEST)->consecutiveFailures); +} + +void test_health_relearn_new_hop_resets_failures(void) +{ + shim->noteRouteLearned(DEST, 0xAB, 1000); + shim->noteRouteFailure(DEST); + shim->noteRouteFailure(DEST); + shim->noteRouteLearned(DEST, 0xCD, 2000); // genuinely new hop -> clean slate + RouteHealth *h = shim->findRouteHealth(DEST); + TEST_ASSERT_EQUAL_UINT8(0, h->consecutiveFailures); + TEST_ASSERT_EQUAL_HEX8(0xCD, h->lastNextHop); +} + +void test_health_failure_without_record_is_noop(void) +{ + shim->noteRouteFailure(DEST); // no record yet + TEST_ASSERT_NULL(shim->findRouteHealth(DEST)); +} + +void test_health_clear(void) +{ + shim->noteRouteLearned(DEST, 0xAB, 1000); + TEST_ASSERT_NOT_NULL(shim->findRouteHealth(DEST)); + shim->clearRouteHealth(DEST); + TEST_ASSERT_NULL(shim->findRouteHealth(DEST)); +} + +void test_health_lru_eviction_bounds_table(void) +{ + // Fill every slot with increasing learn times, then add one more: the oldest must be evicted. + for (uint8_t i = 0; i < HEALTH_MAX; i++) + shim->noteRouteLearned(0x1000 + i, 0xAB, 1000 + (uint32_t)i * 1000); + NodeNum oldest = 0x1000; + TEST_ASSERT_NOT_NULL(shim->findRouteHealth(oldest)); + shim->noteRouteLearned(0x2000, 0xAB, 1000 + (uint32_t)HEALTH_MAX * 1000); // overflow + TEST_ASSERT_NULL(shim->findRouteHealth(oldest)); // evicted + TEST_ASSERT_NOT_NULL(shim->findRouteHealth(0x2000)); // newest present +} + +// =========================================================================== +// Group 4 — shouldDecrementHopLimit favorite-router resolution (M2, site 4) +// =========================================================================== + +void test_hoplimit_preserve_unique_favorite_router(void) +{ + config.device.role = meshtastic_Config_DeviceConfig_Role_ROUTER; + mockNodeDB->addNode(0x000007AB, 0, true, 60, meshtastic_Config_DeviceConfig_Role_ROUTER, /*favorite=*/true); + meshtastic_MeshPacket p = makeRelayedPacket(/*relay=*/0xAB, /*hopsAway=*/1); + TEST_ASSERT_FALSE(shim->shouldDecrementHopLimit(&p)); // preserve +} + +void test_hoplimit_decrement_on_colliding_favorites(void) +{ + // Headline regression: two favorite routers share the relay byte -> ambiguous -> decrement + // (the old "first NodeDB match wins" scan would non-deterministically preserve). + config.device.role = meshtastic_Config_DeviceConfig_Role_ROUTER; + mockNodeDB->addNode(0x000007AB, 0, true, 60, meshtastic_Config_DeviceConfig_Role_ROUTER, /*favorite=*/true); + mockNodeDB->addNode(0x000008AB, 0, true, 60, meshtastic_Config_DeviceConfig_Role_ROUTER, /*favorite=*/true); + meshtastic_MeshPacket p = makeRelayedPacket(0xAB, 1); + TEST_ASSERT_TRUE(shim->shouldDecrementHopLimit(&p)); // decrement +} + +void test_hoplimit_decrement_when_resolved_not_favorite(void) +{ + config.device.role = meshtastic_Config_DeviceConfig_Role_ROUTER; + mockNodeDB->addNode(0x000007AB, 0, true, 60, meshtastic_Config_DeviceConfig_Role_ROUTER, /*favorite=*/false); + meshtastic_MeshPacket p = makeRelayedPacket(0xAB, 1); + TEST_ASSERT_TRUE(shim->shouldDecrementHopLimit(&p)); // unique but not a favorite -> decrement +} + +// =========================================================================== + +void setup() +{ + initializeTestEnvironment(); + UNITY_BEGIN(); + + mockNodeDB = new MockNodeDB(); + shim = new NextHopRouterTestShim(); + nodeDB = mockNodeDB; + + printf("\n=== resolveLastByte (M1) ===\n"); + RUN_TEST(test_resolve_none_when_empty); + RUN_TEST(test_resolve_zero_byte_is_none); + RUN_TEST(test_resolve_unique_neighbor); + RUN_TEST(test_resolve_collision_is_ambiguous); + RUN_TEST(test_resolve_strict_excludes_stale); + RUN_TEST(test_resolve_strict_excludes_far); + RUN_TEST(test_resolve_lenient_includes_favorite_router); + RUN_TEST(test_resolve_lenient_collision_favorite_plus_neighbor); + RUN_TEST(test_resolve_skips_self); + RUN_TEST(test_resolve_skips_ignored); + RUN_TEST(test_resolve_0x00_maps_to_0xFF_and_collides); + RUN_TEST(test_resolve_unique_helper); + + printf("\n=== getNextHop (M2 + M3 decay) ===\n"); + RUN_TEST(test_getnexthop_unique_returns_byte); + RUN_TEST(test_getnexthop_ambiguous_floods); + RUN_TEST(test_getnexthop_vanished_neighbor_floods); + RUN_TEST(test_getnexthop_split_horizon_floods); + RUN_TEST(test_getnexthop_broadcast_is_nullopt); + RUN_TEST(test_getnexthop_decays_stale_route); + + printf("\n=== route-health helpers (M3) ===\n"); + RUN_TEST(test_health_fresh_not_stale); + RUN_TEST(test_health_ttl_expiry); + RUN_TEST(test_health_ttl_rollover_safe); + RUN_TEST(test_health_failure_threshold); + RUN_TEST(test_health_success_resets_failures); + RUN_TEST(test_health_relearn_same_hop_keeps_failures); + RUN_TEST(test_health_relearn_new_hop_resets_failures); + RUN_TEST(test_health_failure_without_record_is_noop); + RUN_TEST(test_health_clear); + RUN_TEST(test_health_lru_eviction_bounds_table); + + printf("\n=== shouldDecrementHopLimit (M2 site 4) ===\n"); + RUN_TEST(test_hoplimit_preserve_unique_favorite_router); + RUN_TEST(test_hoplimit_decrement_on_colliding_favorites); + RUN_TEST(test_hoplimit_decrement_when_resolved_not_favorite); + + exit(UNITY_END()); +} + +void loop() {} diff --git a/test/test_nodedb_blocked/test_main.cpp b/test/test_nodedb_blocked/test_main.cpp index 2bbe9fb6b..715c01323 100644 --- a/test/test_nodedb_blocked/test_main.cpp +++ b/test/test_nodedb_blocked/test_main.cpp @@ -26,17 +26,22 @@ class NodeDBTestShim : public NodeDB void runDemote() { demoteOldestHotNodesToWarm(); } void runCleanup() { cleanupMeshDB(); } + // Read back the role + protected category the warm tier cached for a node. + bool warmMeta(NodeNum n, uint8_t &role, uint8_t &prot) { return warmStore.lookupMeta(n, role, prot); } + void clearHot() { meshNodes->clear(); numMeshNodes = 0; } - void push(NodeNum num, uint32_t lastHeard, bool favorite, bool ignored, bool withUser, bool withKey) + void push(NodeNum num, uint32_t lastHeard, bool favorite, bool ignored, bool withUser, bool withKey, + meshtastic_Config_DeviceConfig_Role role = meshtastic_Config_DeviceConfig_Role_CLIENT) { meshtastic_NodeInfoLite n = meshtastic_NodeInfoLite_init_zero; n.num = num; n.last_heard = lastHeard; + n.role = role; if (favorite) nodeInfoLiteSetBit(&n, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true); if (ignored) @@ -99,6 +104,34 @@ static void test_migration_demotesOldestKeepsKeepersAndSelf(void) TEST_ASSERT_TRUE(warmHasKey(2000 + 3)); // ...but its key kept in the warm tier } +// Eviction carries the device role + protected category into the warm tier. A TRACKER is +// hop-protected but NOT eviction-protected, so it gets demoted with its key; the warm +// record must report role=TRACKER / category=Role. A plain CLIENT carries role=CLIENT/None. +static void test_migration_carriesRoleAndProtectedIntoWarm(void) +{ + db->seedSelf(); + const int extra = MAX_NUM_NODES + 30; // overflow so the oldest non-protected are demoted + for (int i = 1; i <= extra; i++) { + const auto role = (i == 3) ? meshtastic_Config_DeviceConfig_Role_TRACKER : meshtastic_Config_DeviceConfig_Role_CLIENT; + db->push(2000 + i, /*last_heard=*/i, /*favorite=*/false, /*ignored=*/false, /*withUser=*/true, + /*withKey=*/true, role); + } + + db->runDemote(); + + uint8_t role = 0xFF, prot = 0xFF; + // TRACKER (i=3): demoted out of hot, key kept, role + protected carried into warm. + TEST_ASSERT_NULL(db->getMeshNode(2000 + 3)); + TEST_ASSERT_TRUE(warmHasKey(2000 + 3)); + TEST_ASSERT_TRUE(db->warmMeta(2000 + 3, role, prot)); + TEST_ASSERT_EQUAL(meshtastic_Config_DeviceConfig_Role_TRACKER, role); + TEST_ASSERT_EQUAL((uint8_t)WarmProtected::Role, prot); + // CLIENT (i=4): also demoted, carries role=CLIENT / category=None. + TEST_ASSERT_TRUE(db->warmMeta(2000 + 4, role, prot)); + TEST_ASSERT_EQUAL(meshtastic_Config_DeviceConfig_Role_CLIENT, role); + TEST_ASSERT_EQUAL((uint8_t)WarmProtected::None, prot); +} + // Favourite handling: a favourite is never the eviction victim, even when it is // the oldest node in a full hot store. static void test_eviction_preservesFavorite(void) @@ -172,6 +205,7 @@ NDB_TEST_ENTRY void setup() UNITY_BEGIN(); RUN_TEST(test_migration_demotesOldestKeepsKeepersAndSelf); + RUN_TEST(test_migration_carriesRoleAndProtectedIntoWarm); RUN_TEST(test_eviction_preservesFavorite); RUN_TEST(test_ignored_survivesEvictionAndCleanup); RUN_TEST(test_protectedCap_refusesBeyondLimit); diff --git a/test/test_traffic_management/test_main.cpp b/test/test_traffic_management/test_main.cpp index 7631999b3..7f314ae4b 100644 --- a/test/test_traffic_management/test_main.cpp +++ b/test/test_traffic_management/test_main.cpp @@ -1,3 +1,4 @@ +#include "MeshTypes.h" // Include BEFORE TestUtil.h — provides HAS_TRAFFIC_MANAGEMENT (via mesh-pb-constants.h) #include "TestUtil.h" #include #include @@ -10,6 +11,7 @@ #if HAS_TRAFFIC_MANAGEMENT +#include "airtime.h" #include "mesh/CryptoEngine.h" #include "mesh/MeshService.h" #include "mesh/NodeDB.h" @@ -28,6 +30,25 @@ constexpr NodeNum kLocalNode = 0x11111111; constexpr NodeNum kRemoteNode = 0x22222222; constexpr NodeNum kTargetNode = 0x33333333; +// Telemetry hop exhaustion is gated on channel congestion (alterReceived checks +// airTime->isTxAllowedChannelUtil/isTxAllowedAirUtil). Installs a global +// airTime reporting 100% channel utilization for the enclosing scope. +class ScopedBusyAirTime +{ + public: + ScopedBusyAirTime() : previous(airTime) + { + for (uint32_t i = 0; i < CHANNEL_UTILIZATION_PERIODS; i++) + busy.channelUtilization[i] = 10000; // 10 s of airtime per 10 s period + airTime = &busy; + } + ~ScopedBusyAirTime() { airTime = previous; } + + private: + AirTime busy; + AirTime *previous; +}; + class MockNodeDB : public NodeDB { public: @@ -54,6 +75,32 @@ class MockNodeDB : public NodeDB cachedNode.bitfield |= NODEINFO_BITFIELD_HAS_USER_MASK; } + // Role the TMM should see for the cached node (sender-role-aware throttles). + void setCachedNodeRole(meshtastic_Config_DeviceConfig_Role role) { cachedNode.role = role; } + + // Seed a node into the hot-store buffer at index 1 (index 0 is reserved for + // "self"). Respects the fixed-buffer invariant: `meshNodes` is a buffer of + // MAX_NUM_NODES slots with `numMeshNodes` as the logical count — we grow the + // buffer if needed and bump the count, never clear()/push_back() (which would + // shrink it and break NodeDB::resetNodes()'s begin()+1..end() fill). + void setHotNode(NodeNum n, uint8_t nextHop) + { + if (meshNodes->size() < 2) + meshNodes->resize(2); + (*meshNodes)[1] = meshtastic_NodeInfoLite_init_zero; + (*meshNodes)[1].num = n; + (*meshNodes)[1].next_hop = nextHop; + numMeshNodes = 2; + } + + // Evict everything but "self" — simulates the hot DB rolling over. Logical + // count only; the buffer is left intact so the invariant holds. + void rollHotStore() + { + numMeshNodes = 1; + clearCachedNode(); + } + private: bool hasCachedNode = false; NodeNum cachedNodeNum = 0; @@ -121,6 +168,9 @@ static void resetTrafficConfig() config = meshtastic_LocalConfig_init_zero; config.device.role = meshtastic_Config_DeviceConfig_Role_CLIENT; + channelFile = meshtastic_ChannelFile_init_zero; + owner.is_licensed = false; + myNodeInfo.my_node_num = kLocalNode; router = nullptr; @@ -175,6 +225,42 @@ static meshtastic_MeshPacket makePositionPacket(NodeNum from, int32_t lat, int32 return packet; } +static meshtastic_MeshPacket makePositionPacketWithPrecision(NodeNum from, int32_t lat, int32_t lon, uint32_t precisionBits) +{ + meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_POSITION_APP, from, NODENUM_BROADCAST); + meshtastic_Position pos = meshtastic_Position_init_zero; + pos.has_latitude_i = true; + pos.has_longitude_i = true; + pos.latitude_i = lat; + pos.longitude_i = lon; + pos.precision_bits = precisionBits; + + packet.decoded.payload.size = + pb_encode_to_bytes(packet.decoded.payload.bytes, sizeof(packet.decoded.payload.bytes), &meshtastic_Position_msg, &pos); + return packet; +} + +static bool decodePositionPayload(const meshtastic_MeshPacket &packet, meshtastic_Position &out) +{ + out = meshtastic_Position_init_zero; + return pb_decode_from_bytes(packet.decoded.payload.bytes, packet.decoded.payload.size, &meshtastic_Position_msg, &out); +} + +// Primary channel with a well-known single-byte PSK and the (empty -> preset) +// default name, so Channels::isWellKnownChannel(0) is true. +static void installWellKnownPrimaryChannel() +{ + channelFile = meshtastic_ChannelFile_init_zero; + channelFile.channels_count = 1; + channelFile.channels[0].index = 0; + channelFile.channels[0].has_settings = true; + channelFile.channels[0].role = meshtastic_Channel_Role_PRIMARY; + channelFile.channels[0].settings.psk.size = 1; + channelFile.channels[0].settings.psk.bytes[0] = 1; + config.lora.use_preset = true; + config.lora.modem_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_FAST; +} + static meshtastic_MeshPacket makeNodeInfoPacket(NodeNum from, const char *longName, const char *shortName) { meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_NODEINFO_APP, from, NODENUM_BROADCAST); @@ -290,7 +376,7 @@ static void test_tm_rateLimit_dropsOnlyAfterThreshold(void) moduleConfig.traffic_management.rate_limit_window_secs = 60; moduleConfig.traffic_management.rate_limit_max_packets = 3; TrafficManagementModuleTestShim module; - meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_TEXT_MESSAGE_APP, kRemoteNode); + meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode); ProcessMessage r1 = module.handleReceived(packet); ProcessMessage r2 = module.handleReceived(packet); @@ -305,31 +391,6 @@ static void test_tm_rateLimit_dropsOnlyAfterThreshold(void) TEST_ASSERT_EQUAL_UINT32(1, stats.rate_limit_drops); TEST_ASSERT_TRUE(module.ignoreRequestFlag()); } - -/** - * Verify routing/admin traffic is exempt from rate limiting. - * Important because throttling control traffic can destabilize the mesh. - */ -static void test_tm_rateLimit_skipsRoutingAndAdminPorts(void) -{ - moduleConfig.traffic_management.rate_limit_enabled = true; - moduleConfig.traffic_management.rate_limit_window_secs = 60; - moduleConfig.traffic_management.rate_limit_max_packets = 1; - TrafficManagementModuleTestShim module; - meshtastic_MeshPacket routingPacket = makeDecodedPacket(meshtastic_PortNum_ROUTING_APP, kRemoteNode); - meshtastic_MeshPacket adminPacket = makeDecodedPacket(meshtastic_PortNum_ADMIN_APP, kRemoteNode); - - for (int i = 0; i < 4; i++) { - ProcessMessage rr = module.handleReceived(routingPacket); - ProcessMessage ar = module.handleReceived(adminPacket); - TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(rr)); - TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(ar)); - } - - meshtastic_TrafficManagementStats stats = module.getStats(); - TEST_ASSERT_EQUAL_UINT32(0, stats.rate_limit_drops); -} - /** * Verify packets sourced from this node bypass dedup and rate limiting. * Important so local transmissions are not accidentally self-throttled. @@ -650,12 +711,15 @@ static void test_tm_nodeinfo_directResponse_psramMissDoesNotFallbackToNodeDb(voi #endif /** - * Verify relayed telemetry broadcasts are hop-exhausted when enabled. + * Verify relayed telemetry broadcasts are hop-exhausted when enabled AND the + * channel is congested (telemetry exhaustion is gated on channel utilization, + * unlike position exhaustion). * Important to prevent further mesh propagation while still allowing one relay step. */ static void test_tm_alterReceived_exhaustsRelayedTelemetryBroadcast(void) { moduleConfig.traffic_management.exhaust_hop_telemetry = true; + ScopedBusyAirTime busyChannel; TrafficManagementModuleTestShim module; meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode, NODENUM_BROADCAST); packet.hop_start = 5; @@ -677,6 +741,7 @@ static void test_tm_alterReceived_exhaustsRelayedTelemetryBroadcast(void) static void test_tm_alterReceived_skipsLocalAndUnicast(void) { moduleConfig.traffic_management.exhaust_hop_telemetry = true; + ScopedBusyAirTime busyChannel; // congestion satisfied, so only the skip conditions are under test TrafficManagementModuleTestShim module; meshtastic_MeshPacket unicast = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode, kTargetNode); @@ -794,8 +859,11 @@ static void test_tm_positionDedup_precision32_allowsDistinctPositions(void) } /** - * Verify invalid precision=0 is treated as full precision. - * Important so invalid config does not collapse all positions into one fingerprint. + * Verify precision=0 falls back to the default precision (same contract as + * >32: getConfiguredOrDefault + sanitizePositionPrecision treat 0 as unset). + * Important so invalid config does not collapse all positions into one + * fingerprint — positions in different default-precision grid cells must + * still be distinct. */ static void test_tm_positionDedup_precisionZero_allowsDistinctPositions(void) { @@ -805,7 +873,7 @@ static void test_tm_positionDedup_precisionZero_allowsDistinctPositions(void) TrafficManagementModuleTestShim module; meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); - meshtastic_MeshPacket second = makePositionPacket(kRemoteNode, 374221235, -1220845677); + meshtastic_MeshPacket second = makePositionPacket(kRemoteNode, 384221234, -1210845678); ProcessMessage r1 = module.handleReceived(first); ProcessMessage r2 = module.handleReceived(second); @@ -857,11 +925,11 @@ static void test_tm_positionDedup_priorRateState_doesNotDropFirstFingerprintZero moduleConfig.traffic_management.rate_limit_max_packets = 10; TrafficManagementModuleTestShim module; - meshtastic_MeshPacket text = makeDecodedPacket(meshtastic_PortNum_TEXT_MESSAGE_APP, kRemoteNode); + meshtastic_MeshPacket telemetry = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode); meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 0x12300000, 0x45600000); meshtastic_MeshPacket duplicate = makePositionPacket(kRemoteNode, 0x12300000, 0x45600000); - ProcessMessage seeded = module.handleReceived(text); + ProcessMessage seeded = module.handleReceived(telemetry); ProcessMessage r1 = module.handleReceived(first); ProcessMessage r2 = module.handleReceived(duplicate); meshtastic_TrafficManagementStats stats = module.getStats(); @@ -882,7 +950,7 @@ static void test_tm_rateLimit_resetsAfterWindowExpires(void) moduleConfig.traffic_management.rate_limit_window_secs = 1; moduleConfig.traffic_management.rate_limit_max_packets = 1; TrafficManagementModuleTestShim module; - meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_TEXT_MESSAGE_APP, kRemoteNode); + meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode); ProcessMessage r1 = module.handleReceived(packet); ProcessMessage r2 = module.handleReceived(packet); @@ -895,30 +963,6 @@ static void test_tm_rateLimit_resetsAfterWindowExpires(void) TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(r3)); TEST_ASSERT_EQUAL_UINT32(1, stats.rate_limit_drops); } - -/** - * Verify rate-limit thresholds above 255 effectively clamp to 255. - * Important because counters are uint8_t and must not overflow behavior. - */ -static void test_tm_rateLimit_thresholdAbove255_clamps(void) -{ - moduleConfig.traffic_management.rate_limit_enabled = true; - moduleConfig.traffic_management.rate_limit_window_secs = 60; - moduleConfig.traffic_management.rate_limit_max_packets = 300; - TrafficManagementModuleTestShim module; - meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_TEXT_MESSAGE_APP, kRemoteNode); - - for (int i = 0; i < 255; i++) { - ProcessMessage result = module.handleReceived(packet); - TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(result)); - } - ProcessMessage dropped = module.handleReceived(packet); - meshtastic_TrafficManagementStats stats = module.getStats(); - - TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::STOP), static_cast(dropped)); - TEST_ASSERT_EQUAL_UINT32(1, stats.rate_limit_drops); -} - /** * Verify unknown-packet tracking resets after its active window expires. * Important so old unknown traffic does not trigger delayed drops. @@ -966,12 +1010,14 @@ static void test_tm_unknownPackets_thresholdAbove255_clamps(void) } /** - * Verify relayed position broadcasts can also be hop-exhausted. + * Verify relayed position broadcasts can also be hop-exhausted — under the + * same pressure gate as telemetry (here: channel congestion). * Important because telemetry and position use separate exhaust flags. */ static void test_tm_alterReceived_exhaustsRelayedPositionBroadcast(void) { moduleConfig.traffic_management.exhaust_hop_position = true; + ScopedBusyAirTime busyChannel; TrafficManagementModuleTestShim module; meshtastic_MeshPacket packet = makePositionPacket(kRemoteNode, 374221234, -1220845678, NODENUM_BROADCAST); packet.hop_start = 5; @@ -985,7 +1031,6 @@ static void test_tm_alterReceived_exhaustsRelayedPositionBroadcast(void) TEST_ASSERT_TRUE(module.shouldExhaustHops(packet)); TEST_ASSERT_EQUAL_UINT32(1, stats.hop_exhausted_packets); } - /** * Verify hop exhaustion ignores undecoded/encrypted packets. * Important so we never mutate packets that were not decoded by this module. @@ -993,6 +1038,7 @@ static void test_tm_alterReceived_exhaustsRelayedPositionBroadcast(void) static void test_tm_alterReceived_skipsUndecodedPackets(void) { moduleConfig.traffic_management.exhaust_hop_telemetry = true; + ScopedBusyAirTime busyChannel; // congestion satisfied, so only the undecoded skip is under test TrafficManagementModuleTestShim module; meshtastic_MeshPacket packet = makeUnknownPacket(kRemoteNode, NODENUM_BROADCAST); packet.hop_start = 5; @@ -1014,6 +1060,7 @@ static void test_tm_alterReceived_skipsUndecodedPackets(void) static void test_tm_alterReceived_resetExhaustFlagOnNextPacket(void) { moduleConfig.traffic_management.exhaust_hop_telemetry = true; + ScopedBusyAirTime busyChannel; // telemetry exhaust only fires under congestion TrafficManagementModuleTestShim module; meshtastic_MeshPacket telemetry = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode, NODENUM_BROADCAST); @@ -1039,6 +1086,7 @@ static void test_tm_alterReceived_resetExhaustFlagOnNextPacket(void) static void test_tm_alterReceived_exhaustFlag_isPacketScoped(void) { moduleConfig.traffic_management.exhaust_hop_telemetry = true; + ScopedBusyAirTime busyChannel; // telemetry exhaust only fires under congestion TrafficManagementModuleTestShim module; meshtastic_MeshPacket exhausted = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode, NODENUM_BROADCAST); @@ -1083,6 +1131,93 @@ static void test_tm_runOnce_enabledReturnsMaintenanceInterval(void) TEST_ASSERT_EQUAL_INT32(60 * 1000, interval); } +// --------------------------------------------------------------------------- +// Next-hop overflow cache +// --------------------------------------------------------------------------- + +/** + * Round-trip set/get of a confirmed next hop, plus the input guards. + */ +static void test_tm_nextHop_setAndGetRoundTrip(void) +{ + TrafficManagementModuleTestShim module; + + // Unknown node yields no hint. + TEST_ASSERT_EQUAL_UINT8(0, module.getNextHopHint(kTargetNode)); + + // Store a confirmed hop and read it back. + module.setNextHop(kTargetNode, 0x42); + TEST_ASSERT_EQUAL_UINT8(0x42, module.getNextHopHint(kTargetNode)); + + // Zero dest and zero byte are rejected (no spurious entry created). + module.setNextHop(0, 0x42); + module.setNextHop(kRemoteNode, 0); + TEST_ASSERT_EQUAL_UINT8(0, module.getNextHopHint(kRemoteNode)); + + // Last-write-wins on re-confirmation. + module.setNextHop(kTargetNode, 0x99); + TEST_ASSERT_EQUAL_UINT8(0x99, module.getNextHopHint(kTargetNode)); +} + +/** + * The headline scenario: a node carrying a next hop in the hot NodeInfoLite DB + * is warm-loaded into the TMM cache, then the hot DB is "rolled" (the node ages + * out entirely). The hint must still be served — now exclusively from TMM. + */ +static void test_tm_nextHop_servedAfterNodeDbRoll(void) +{ + TrafficManagementModuleTestShim module; + + // Seed the hot NodeInfoLite DB with a node that has a confirmed next hop. + mockNodeDB->setHotNode(kTargetNode, 0x42); + + // Warm-start the overflow cache from the hot DB. + module.preloadNextHopsFromNodeDB(); + TEST_ASSERT_EQUAL_UINT8(0x42, module.getNextHopHint(kTargetNode)); + + // Roll the main NodeInfoLite DB: the node is evicted from the hot store. + mockNodeDB->rollHotStore(); + TEST_ASSERT_NULL(nodeDB->getMeshNode(kTargetNode)); // gone from the hot store + + // Hit is still served — proving it now comes from the TMM overflow cache. + TEST_ASSERT_EQUAL_UINT8(0x42, module.getNextHopHint(kTargetNode)); +} + +/** + * Preload must not clobber a freshly-learned (confirmed) hop with a possibly + * stale persisted one from NodeInfoLite. + */ +static void test_tm_nextHop_preloadDoesNotClobberLearned(void) +{ + TrafficManagementModuleTestShim module; + + // A fresher confirmed hop is already cached. + module.setNextHop(kTargetNode, 0x99); + + // The hot DB carries an older next hop for the same node. + mockNodeDB->setHotNode(kTargetNode, 0x42); + + module.preloadNextHopsFromNodeDB(); + + // The freshly-learned hop survives. + TEST_ASSERT_EQUAL_UINT8(0x99, module.getNextHopHint(kTargetNode)); +} + +/** + * A pure routing hint (no dedup/rate/unknown state) must survive the maintenance + * sweep — next_hop != 0 keeps the slot alive even though it has no TTL. + */ +static void test_tm_nextHop_keptAliveAcrossMaintenanceSweep(void) +{ + TrafficManagementModuleTestShim module; + + module.setNextHop(kTargetNode, 0x42); + + // The sweep frees slots whose sub-stores are all empty; next_hop must veto that. + module.runOnce(); + + TEST_ASSERT_EQUAL_UINT8(0x42, module.getNextHopHint(kTargetNode)); +} } // namespace void setUp(void) @@ -1106,7 +1241,6 @@ TM_TEST_ENTRY void setup() RUN_TEST(test_tm_positionDedup_dropsDuplicateWithinWindow); RUN_TEST(test_tm_positionDedup_allowsMovedPosition); RUN_TEST(test_tm_rateLimit_dropsOnlyAfterThreshold); - RUN_TEST(test_tm_rateLimit_skipsRoutingAndAdminPorts); RUN_TEST(test_tm_fromUs_bypassesPositionAndRateFilters); RUN_TEST(test_tm_localDestination_bypassesTransitFilters); RUN_TEST(test_tm_nodeinfo_routerClamp_skipsWhenTooManyHops); @@ -1130,7 +1264,6 @@ TM_TEST_ENTRY void setup() RUN_TEST(test_tm_positionDedup_epochReset_doesNotDropFirstPacketAfterReset); RUN_TEST(test_tm_positionDedup_priorRateState_doesNotDropFirstFingerprintZero); RUN_TEST(test_tm_rateLimit_resetsAfterWindowExpires); - RUN_TEST(test_tm_rateLimit_thresholdAbove255_clamps); RUN_TEST(test_tm_unknownPackets_resetAfterWindowExpires); RUN_TEST(test_tm_unknownPackets_thresholdAbove255_clamps); RUN_TEST(test_tm_alterReceived_exhaustsRelayedPositionBroadcast); @@ -1139,6 +1272,10 @@ TM_TEST_ENTRY void setup() RUN_TEST(test_tm_alterReceived_exhaustFlag_isPacketScoped); RUN_TEST(test_tm_runOnce_disabledReturnsMaxInterval); RUN_TEST(test_tm_runOnce_enabledReturnsMaintenanceInterval); + RUN_TEST(test_tm_nextHop_setAndGetRoundTrip); + RUN_TEST(test_tm_nextHop_servedAfterNodeDbRoll); + RUN_TEST(test_tm_nextHop_preloadDoesNotClobberLearned); + RUN_TEST(test_tm_nextHop_keptAliveAcrossMaintenanceSweep); exit(UNITY_END()); } diff --git a/test/test_warm_store/test_main.cpp b/test/test_warm_store/test_main.cpp index 3c8455883..2ecacc75d 100644 --- a/test/test_warm_store/test_main.cpp +++ b/test/test_warm_store/test_main.cpp @@ -13,8 +13,10 @@ #if WARM_NODE_COUNT > 0 +#include "FSCommon.h" #include "mesh/WarmNodeStore.h" #include +#include namespace { @@ -76,12 +78,15 @@ void test_ws_take_removesEntry() WarmNodeStore ws; uint8_t key[32]; makeKey(key, 3); - ws.absorb(0x400, 1234, key); + ws.absorb(0x400, 1234, key, 5 /* TRACKER */, (uint8_t)WarmProtected::Role); WarmNodeEntry e; TEST_ASSERT_TRUE(ws.take(0x400, e)); TEST_ASSERT_EQUAL(0x400, e.num); - TEST_ASSERT_EQUAL(1234, e.last_heard); + // last_heard is quantised to the metadata quantum; role/protected ride the low bits. + TEST_ASSERT_EQUAL(1234u & WARM_TIME_MASK, warmTimeOf(e)); + TEST_ASSERT_EQUAL(5, warmRoleOf(e)); + TEST_ASSERT_EQUAL((uint8_t)WarmProtected::Role, warmProtOf(e)); TEST_ASSERT_EQUAL_MEMORY(key, e.public_key, 32); TEST_ASSERT_FALSE(ws.contains(0x400)); TEST_ASSERT_FALSE(ws.take(0x400, e)); @@ -111,13 +116,15 @@ void test_ws_keyedCandidate_evictsOldestKeylessFirst() // Fill with keyed entries except two keyless ones in the middle for (size_t i = 0; i < ws.capacity(); i++) { const bool keyless = (i == 5 || i == 10); - TEST_ASSERT_TRUE(ws.absorb(0x1000 + i, keyless ? (i == 10 ? 50 : 60) : 10, keyless ? NULL : key)); + // Timestamps spaced by the 64 s warm metadata quantum (<<6) so LRU order survives + // quantisation: keyless i=10 is oldest (50), i=5 next (60), keyed all older (10). + TEST_ASSERT_TRUE(ws.absorb(0x1000 + i, (keyless ? (i == 10 ? 50u : 60u) : 10u) << 6, keyless ? NULL : key)); } // Keyed candidate must displace the OLDEST KEYLESS entry (0x100A, ts=50), // even though every keyed entry is older (ts=10) uint8_t k2[32]; makeKey(k2, 0x43); - TEST_ASSERT_TRUE(ws.absorb(0x8888, 70, k2)); + TEST_ASSERT_TRUE(ws.absorb(0x8888, 70u << 6, k2)); TEST_ASSERT_FALSE(ws.contains(0x1000 + 10)); TEST_ASSERT_TRUE(ws.contains(0x1000 + 5)); TEST_ASSERT_TRUE(ws.contains(0x8888)); @@ -139,6 +146,31 @@ void test_ws_keyedCandidate_evictsOldestKeyedWhenNoKeyless() TEST_ASSERT_EQUAL(ws.capacity(), ws.count()); } +void test_ws_meta_roundTrip() +{ + WarmNodeStore ws; + uint8_t key[32]; + makeKey(key, 0x77); + // Keyed TRACKER(5)/Role-protected and keyless SENSOR(6)/unprotected. + TEST_ASSERT_TRUE(ws.absorb(0x700, 1234, key, 5 /* TRACKER */, (uint8_t)WarmProtected::Role)); + TEST_ASSERT_TRUE(ws.absorb(0x701, 5678, NULL, 6 /* SENSOR */, (uint8_t)WarmProtected::None)); + + uint8_t role = 0xFF, prot = 0xFF; + TEST_ASSERT_TRUE(ws.lookupMeta(0x700, role, prot)); + TEST_ASSERT_EQUAL(5, role); + TEST_ASSERT_EQUAL((uint8_t)WarmProtected::Role, prot); + TEST_ASSERT_TRUE(ws.lookupMeta(0x701, role, prot)); + TEST_ASSERT_EQUAL(6, role); + TEST_ASSERT_EQUAL((uint8_t)WarmProtected::None, prot); + // Absent node yields false and leaves outputs untouched-by-contract (just check return). + TEST_ASSERT_FALSE(ws.lookupMeta(0x999, role, prot)); + // Default args still compile (role/protected = 0 = CLIENT/None). + TEST_ASSERT_TRUE(ws.absorb(0x702, 9999, NULL)); + TEST_ASSERT_TRUE(ws.lookupMeta(0x702, role, prot)); + TEST_ASSERT_EQUAL(0, role); + TEST_ASSERT_EQUAL((uint8_t)WarmProtected::None, prot); +} + void test_ws_remove_and_clear() { WarmNodeStore ws; @@ -175,6 +207,56 @@ void test_ws_persistence_roundTrip() b.saveIfDirty(); } +// Migration: a v1 (WRM1) warm.dat must keep identity + key but discard last_heard +// (so its low bits aren't misread as role/protected). File backend only. +void test_ws_v1_migration_discardsLastHeard() +{ + WarmNodeStore a; + uint8_t key[32], got[32]; + makeKey(key, 0x66); + a.absorb(0x900, 123456, key, 5 /* TRACKER */, (uint8_t)WarmProtected::Role); + if (!a.saveIfDirty()) { + TEST_IGNORE_MESSAGE("Filesystem not available in this test environment"); + return; + } + + // Read the whole v2 file, flip the 4-byte header magic to v1 ("WRM1"), write it back. + // (CRC covers only the entry bytes, so patching the header magic keeps it valid.) + std::vector buf; + { + auto f = FSCom.open("/prefs/warm.dat", FILE_O_READ); + if (!f) { + TEST_IGNORE_MESSAGE("warm.dat not readable in this environment"); + return; + } + buf.resize(f.size()); + f.read(buf.data(), buf.size()); + f.close(); + } + TEST_ASSERT_TRUE(buf.size() >= 4); + const uint32_t v1magic = 0x314D5257u; // "WRM1" + memcpy(buf.data(), &v1magic, sizeof(v1magic)); + { + auto f = FSCom.open("/prefs/warm.dat", FILE_O_WRITE); + TEST_ASSERT_TRUE((bool)f); + f.write(buf.data(), buf.size()); + f.close(); + } + + WarmNodeStore b; + b.load(); + TEST_ASSERT_TRUE(b.contains(0x900)); // identity survived migration + TEST_ASSERT_TRUE(b.copyKey(0x900, got)); // public key survived + TEST_ASSERT_EQUAL_MEMORY(key, got, 32); + uint8_t role = 0xFF, prot = 0xFF; + TEST_ASSERT_TRUE(b.lookupMeta(0x900, role, prot)); + TEST_ASSERT_EQUAL(0, role); // last_heard discarded → role/protected reset + TEST_ASSERT_EQUAL((uint8_t)WarmProtected::None, prot); + + b.clear(); + b.saveIfDirty(); +} + WS_TEST_ENTRY void setup() { initializeTestEnvironment(); @@ -187,8 +269,10 @@ WS_TEST_ENTRY void setup() RUN_TEST(test_ws_keylessCandidate_neverEvictsKeyedEntries); RUN_TEST(test_ws_keyedCandidate_evictsOldestKeylessFirst); RUN_TEST(test_ws_keyedCandidate_evictsOldestKeyedWhenNoKeyless); + RUN_TEST(test_ws_meta_roundTrip); RUN_TEST(test_ws_remove_and_clear); RUN_TEST(test_ws_persistence_roundTrip); + RUN_TEST(test_ws_v1_migration_discardsLastHeard); exit(UNITY_END()); } From dd1ec9d462e98061b8993b42ec683926c0fe07dc Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Fri, 19 Jun 2026 19:56:24 -0500 Subject: [PATCH 19/86] Lora region preset map (#10736) * Added lora region and preset maps * Protos * Address PR review feedback - Log (and break/skip) when the region preset map exceeds its array bounds instead of silently dropping regions - Derive test bounds from the generated nanopb array sizes via sizeof() instead of hard-coded magic numbers - Fix want_config sequence comment (missing comma, STATE_SEND_MODULECONFIG) - Specify a language on the spec's fenced code blocks (markdownlint MD040) * Fix want_config stall: handle STATE_SEND_REGION_PRESETS in PhoneAPI::available() available() had a separate per-state switch that wasn't updated for the new state, so it returned false ('unexpected state 5') and getFromRadio() was never called - the config handshake stalled after metadata and the client timed out. Verified via the native simulator integration test. --- ...region_preset_compatibility_client_spec.md | 277 ++++++++++++++++++ protobufs | 2 +- src/mesh/MeshRadio.h | 5 + src/mesh/PhoneAPI.cpp | 20 +- src/mesh/PhoneAPI.h | 1 + src/mesh/RadioInterface.cpp | 56 ++++ src/mesh/generated/meshtastic/mesh.pb.cpp | 9 + src/mesh/generated/meshtastic/mesh.pb.h | 107 ++++++- test/test_radio/test_main.cpp | 83 ++++++ 9 files changed, 556 insertions(+), 4 deletions(-) create mode 100644 docs/lora_region_preset_compatibility_client_spec.md diff --git a/docs/lora_region_preset_compatibility_client_spec.md b/docs/lora_region_preset_compatibility_client_spec.md new file mode 100644 index 000000000..b62fe7757 --- /dev/null +++ b/docs/lora_region_preset_compatibility_client_spec.md @@ -0,0 +1,277 @@ +# LoRa Region → Preset Compatibility — Client Implementation Spec + +**Status:** Draft for 2.8 · **Audience:** Meshtastic client app developers (Android first, +Apple second, then web/python) · **Firmware side:** implemented in `firmware` +(`FromRadio.region_presets`, see below). + +> This document lives in the firmware repo while the feature is developed. It is meant to +> graduate to `meshtastic/protobufs` (and/or the docs site) alongside the upstream protobuf +> PR that reserves `FromRadio` field **19**. + +--- + +## 1. Why this exists + +For 2.8 the LoRa regions and modem presets were reworked. **Not every modem preset is legal +in every region** — narrow EU SRD bands, the EU 868 "narrow" band, amateur/ham bands, and +the 2.4 GHz band each accept only a specific subset of presets. The firmware already +enforces this internally (it clamps or rejects illegal combinations), but until now a client +had no way to _know_ the rules, so a user could pick an illegal region+preset pair in the UI +and only discover the problem after the device silently corrected it. + +This feature has the firmware **declare the legal region→preset combinations** to the client +during the `want_config` handshake, so the client UI can constrain the preset picker to the +valid set for the currently selected region (and warn about licensed-only bands). It is +purely advisory metadata — the firmware remains the source of truth and still +validates/clamps on its own. + +--- + +## 2. Protocol additions + +Three new messages in `meshtastic/mesh.proto`, plus one new `FromRadio` oneof variant. + +### 2.1 `FromRadio.region_presets` (field 19) + +```proto +message FromRadio { + uint32 id = 1; + oneof payload_variant { + // ... fields 2..18 unchanged ... + LoRaRegionPresetMap region_presets = 19; + } +} +``` + +### 2.2 Messages + +```proto +// A distinct set of legal modem presets shared by one or more LoRa regions. +message LoRaPresetGroup { + repeated Config.LoRaConfig.ModemPreset presets = 1; // legal presets for this group + Config.LoRaConfig.ModemPreset default_preset = 2; // always one of `presets` + bool licensed_only = 3; // ham/amateur band → warn/gate +} + +// Associates a single LoRa region with its preset group (by index). +message LoRaRegionPresets { + Config.LoRaConfig.RegionCode region = 1; + uint32 group_index = 2; // index into LoRaRegionPresetMap.groups +} + +// The full map, delivered grouped to fit one FromRadio packet. +message LoRaRegionPresetMap { + repeated LoRaPresetGroup groups = 1; // each distinct preset list + repeated LoRaRegionPresets region_groups = 2; // every known region → a group index +} +``` + +### 2.3 Why grouped (and the size envelope clients should respect) + +A `FromRadio` packet is capped at **512 bytes** (`MAX_TO_FROM_RADIO_SIZE`). Most regions +share one identical preset list (the "standard" 9-preset list), so the map is delivered +**grouped**: `groups` holds each _distinct_ preset list once, and `region_groups` maps every +known region to one of those groups by index. This keeps the encoded size additive +(`groups` + `region_groups`) rather than multiplicative, well under the cap. + +nanopb (firmware) array bounds — clients do **not** need to enforce these, but they bound +what you can receive: + +| field | max_count | +| ----------------------------------- | ------------------------------------ | +| `LoRaRegionPresetMap.groups` | 8 | +| `LoRaRegionPresetMap.region_groups` | 38 (= number of `RegionCode` values) | +| `LoRaPresetGroup.presets` | 11 | + +--- + +## 3. When it is delivered + +`region_presets` is sent **once** during the `want_config` handshake, as a single +`FromRadio` message, in this position: + +```text +my_info → (deviceuiConfig) → node_info(self) → metadata → region_presets → channel… → config… → moduleConfig… → node_info(others)… → fileInfo… → config_complete_id → (live packets) +``` + +i.e. **immediately after `metadata` and before the first `channel`**. + +- It is included for a normal full `want_config` and for the **config-only** nonce. +- It is **omitted** for the **nodes-only** nonce (that path skips metadata/config entirely). +- A client must **not** assume it always arrives (see §5). + +--- + +## 4. Decoding into a usable lookup + +Flatten the grouped wire form into `Map`: + +```text +struct RegionPresetInfo { Set presets; ModemPreset default; bool licensedOnly } + +fun decode(map: LoRaRegionPresetMap): Map { + result = {} + for (rg in map.region_groups) { + if (rg.group_index >= map.groups.size) continue // defensive: malformed/forward data + g = map.groups[rg.group_index] + result[rg.region] = RegionPresetInfo( + presets = g.presets.toSet(), + default = g.default_preset, + licensedOnly = g.licensed_only) + } + return result +} +``` + +Persist this map alongside the rest of the downloaded config so the LoRa config screen can +read it synchronously. + +--- + +## 5. Semantics & rules (the load-bearing part) + +These rules are what keep the UX correct across firmware versions. Implement all of them. + +1. **Absent region ⇒ no constraint.** If a `RegionCode` does not appear in `region_groups`, + the client has _no_ compatibility info for it and **must not restrict** its preset + choices (fall back to allowing the full `ModemPreset` list). This happens for a handful + of `RegionCode` enum values that have no firmware band table entry (today: `EU_874`, + `EU_917`, `ITU1_70CM`, `ITU2_70CM`, `ITU3_70CM`). + +2. **Absent message ⇒ no constraint.** Firmware older than 2.8 never sends `region_presets`. + New clients **must** tolerate the message being absent entirely and keep their existing + (unconstrained) behavior. Do not block the config screen waiting for it. + +3. **`default_preset`** is always a member of that group's `presets`. Use it to pre-select a + preset when the user switches to a region whose valid set does not include the currently + selected preset (instead of leaving an illegal selection or guessing). + +4. **`licensed_only`** marks ham/amateur bands. Surface a warning or gate (the firmware also + requires the operator's `is_licensed` flag for these regions; coordinate the two so the + user isn't allowed to pick a licensed band without acknowledging licensing). + +5. **EU region auto-swap caveat.** The firmware treats the EU sibling regions + (`EU_868` / `EU_866` / `EU_N_868`) specially: if the user is in one of them and selects a + preset that belongs to a sibling's list, the firmware **swaps the region** rather than + rejecting the preset. Consequence for clients: **do not assume the region is immutable + across a preset change** — after an admin config write, re-read the resulting + `LoRaConfig` and reflect the (possibly changed) region back into the UI. + +6. **Use it as a UI guard, not a validator of truth.** The firmware still validates/clamps + on its own. The map exists to prevent the user from _selecting_ an illegal combo; it is + not a security or correctness boundary. + +--- + +## 6. UI/UX recommendations + +- In the LoRa config screen, when a region is selected, **filter/enable the modem-preset + picker to that region's `presets`** (when `use_preset`/`use_modem_preset` is on). +- If the current preset is not in the newly selected region's set, switch the selection to + that region's `default_preset`. +- Show a **licensed badge / confirmation** for regions where `licensed_only == true`. +- If a region is absent from the map (rule §5.1) or the whole message is absent (§5.2), + render the full preset list as before — never show an empty picker. + +--- + +## 7. Forward / backward compatibility + +- **Old clients, new firmware:** an unknown `FromRadio` oneof variant (field 19) is ignored + by protobuf/nanopb decoders; the relative ordering of the known messages is unchanged, so + existing apps are unaffected. +- **New clients, old firmware:** message simply never arrives → treat as "no constraints" + (§5.2). +- **Enum growth:** new `RegionCode`/`ModemPreset` values may appear over time. Decoders + should pass through unknown enum values rather than crashing; an unknown region in + `region_groups` is harmless (the client just won't have a localized name for it). + +--- + +## 8. Platform notes + +> Verified against the `main` branch of each repo. Both have been refactored away from +> older layouts; re-pin file paths against a specific commit if you need them durable. + +### 8.1 Android — `meshtastic/Meshtastic-Android` (Kotlin / Compose, KMP) + +- **Protobufs are a published Maven artifact, _not_ a submodule.** Declared in + `gradle/libs.versions.toml` (`org.meshtastic:protobufs`, currently `2.7.25`); generated + package is **`org.meshtastic.proto`**. **A `region_presets`-aware build requires a new + published `org.meshtastic:protobufs` release**, then bumping that one version string. +- **The protobufs are Wire-generated**, so the `FromRadio` oneof is **not** a + `payloadVariantCase` enum — each arm is a **nullable field**. Handle the new variant in + `FromRadioPacketHandlerImpl.handleFromRadio(...)` + (`core/data/.../manager/FromRadioPacketHandlerImpl.kt`) by adding a + `regionPresets != null -> …` arm to the existing `when { … }`, delegating to a handler + (mirror `handleLocalMetadata` / `handleConfigComplete`). +- **State holder:** expose the decoded map from `RadioConfigRepository` / + `RadioConfigRepositoryImpl` as a `Flow` (mirroring `localConfigFlow`/`channelSetFlow`), + consumed by `feature/settings/.../radio/RadioConfigViewModel.kt`. +- **UI:** the region & preset dropdowns are `DropDownPreference`s in + `feature/settings/.../radio/component/LoRaConfigItemList.kt` (public composable + `LoRaConfigScreen`). Gate/filter the `ChannelOption` (preset) dropdown by the selected + `RegionInfo`'s entry in the map. + +### 8.2 Apple — `meshtastic/Meshtastic-Apple` (Swift / SwiftUI) + +- **Protobufs are vendored** into a local Swift package `MeshtasticProtobufs` + (`MeshtasticProtobufs/Sources/meshtastic/*.pb.swift`), generated from the `protobufs` git + submodule via `scripts/gen_protos.sh`. **To get field 19:** advance the `protobufs` + submodule, run `scripts/gen_protos.sh`, commit the regenerated `.pb.swift` + submodule + pointer. (No published-artifact dependency — Apple can regenerate from any commit.) +- **Dispatch:** `AccessoryManager.processFromRadio(_:)` + (`Meshtastic/Accessory/Accessory Manager/AccessoryManager.swift`) is a real + `switch decodedInfo.payloadVariant { … }` — add a `.regionPresets` case, with the handler + in `AccessoryManager+FromRadio.swift` (mirror `handleConfig` / `handleMetadata`). +- **Persistence:** config is **SwiftData** (`@Model` entities), upserted via + `MeshPackets`/`UpdateSwiftData.swift`. Store the decoded map (e.g. on a settings/connection + model) so the LoRa view can read it. +- **UI:** `Meshtastic/Views/Settings/Config/LoRaConfig.swift` (`struct LoRaConfig: View`) + has the `Picker("Region", …)` (`RegionCodes.userSelectable`) and `Picker("Presets", …)` + (`ModemPresets.userSelectable`, gated on `usePreset`). Filter the presets picker by the + selected region's entry. Enums live in `Meshtastic/Enums/LoraConfigEnums.swift`. + +### 8.3 Other clients + +- **python (`meshtastic` / Meshtastic-python)** and **web** consume the published protobufs; + they will see `region_presets` once their protobuf dependency includes field 19, and can + ignore it until then (it decodes as an unknown field). + +--- + +## 9. Reference payload (current firmware table) + +For decoder unit tests. With the 2.8 region table, the firmware emits **6 groups**. Group +indices are assigned in region-table order (first region to use a profile creates its group), +so they are stable as listed here: + +| group_index | default_preset | licensed_only | presets | +| ----------------------- | -------------- | ------------- | -------------------------------------------------------------------------------------------------------------- | +| 0 (standard) | `LONG_FAST` | false | LONG_FAST, LONG_SLOW, MEDIUM_SLOW, MEDIUM_FAST, SHORT_SLOW, SHORT_FAST, LONG_MODERATE, SHORT_TURBO, LONG_TURBO | +| 1 (EU 868) | `LONG_FAST` | false | LONG_FAST, LONG_SLOW, MEDIUM_SLOW, MEDIUM_FAST, SHORT_SLOW, SHORT_FAST, LONG_MODERATE | +| 2 (EU 866 SRD / "lite") | `LITE_FAST` | false | LITE_FAST, LITE_SLOW | +| 3 (EU 868 narrow) | `NARROW_SLOW` | false | NARROW_FAST, NARROW_SLOW | +| 4 (ham 20 kHz) | `TINY_FAST` | **true** | TINY_FAST, TINY_SLOW | +| 5 (ham 100 kHz) | `NARROW_SLOW` | **true** | NARROW_FAST, NARROW_SLOW | + +`region_groups` (region → group_index): + +| group | regions | +| ----- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 0 | US, EU_433, CN, JP, ANZ, ANZ_433, RU, KR, TW, IN, NZ_865, TH, UA_433, UA_868, MY_433, MY_919, SG_923, PH_433, PH_868, PH_915, KZ_433, KZ_863, NP_865, BR_902, LORA_24 | +| 1 | EU_868 | +| 2 | EU_866 | +| 3 | EU_N_868 | +| 4 | ITU1_2M, ITU2_2M, ITU3_2M | +| 5 | ITU2_125CM | + +> Note groups **3** and **5** carry the same preset list (NARROW\_\*) but are distinct groups +> because they differ in `licensed_only`. Decoders must key on the group, not on the preset +> list, to preserve the licensing flag. +> +> Regions **absent** from the table (no constraint info; see §5.1): `EU_874`, `EU_917`, +> `ITU1_70CM`, `ITU2_70CM`, `ITU3_70CM`. + +This table is generated from the firmware's region table at runtime; treat the firmware as +authoritative and these values as the expected snapshot for the 2.8 table. diff --git a/protobufs b/protobufs index 362516671..f16a13e43 160000 --- a/protobufs +++ b/protobufs @@ -1 +1 @@ -Subproject commit 36251667179496c1e1261f4e4e5b349a51e5e861 +Subproject commit f16a13e43393f3c6a88a1da36e75dfc29f448d3e diff --git a/src/mesh/MeshRadio.h b/src/mesh/MeshRadio.h index 965f3c46e..3387887c2 100644 --- a/src/mesh/MeshRadio.h +++ b/src/mesh/MeshRadio.h @@ -88,6 +88,11 @@ extern const RegionInfo *myRegion; extern void initRegion(); extern const RegionInfo *getRegion(meshtastic_Config_LoRaConfig_RegionCode code); +// Fill `map` with the region->valid-preset table, grouped so regions sharing a +// preset list reference the same group. Sent to clients during want_config so +// their UI can block illegal region+preset combinations. +extern void getRegionPresetMap(meshtastic_LoRaRegionPresetMap &map); + // Valid LoRa spread factor range and defaults constexpr uint8_t LORA_SF_MIN = 5; constexpr uint8_t LORA_SF_MAX = 12; diff --git a/src/mesh/PhoneAPI.cpp b/src/mesh/PhoneAPI.cpp index d37a3e837..5ee553bdc 100644 --- a/src/mesh/PhoneAPI.cpp +++ b/src/mesh/PhoneAPI.cpp @@ -12,6 +12,7 @@ #include "Channels.h" #include "Default.h" #include "FSCommon.h" +#include "MeshRadio.h" #include "MeshService.h" #include "NodeDB.h" #include "PacketHistory.h" @@ -516,9 +517,10 @@ bool PhoneAPI::handleToRadio(const uint8_t *buf, size_t bufLength) STATE_SEND_UIDATA, STATE_SEND_OWN_NODEINFO, STATE_SEND_METADATA, - STATE_SEND_CHANNELS + STATE_SEND_REGION_PRESETS, // region -> valid modem presets (one message) + STATE_SEND_CHANNELS, STATE_SEND_CONFIG, - STATE_SEND_MODULE_CONFIG, + STATE_SEND_MODULECONFIG, STATE_SEND_OTHER_NODEINFOS, // states progress in this order as the device sends to the client STATE_SEND_FILEMANIFEST, STATE_SEND_COMPLETE_ID, @@ -636,7 +638,20 @@ size_t PhoneAPI::getFromRadio(uint8_t *buf) memset(&fromRadioScratch.metadata, 0, sizeof(fromRadioScratch.metadata)); } #endif + state = STATE_SEND_REGION_PRESETS; + break; + + case STATE_SEND_REGION_PRESETS: + // Tell the client which modem presets are legal in each region so its UI + // can block illegal region+preset combinations. This is public RF / + // regulatory information (region and modem_preset are already in the + // unauthenticated LoRa whitelist below), so it is sent unconditionally — + // even an unauthorized/locked-down client can render a correct picker. + LOG_DEBUG("Send region preset map"); + fromRadioScratch.which_payload_variant = meshtastic_FromRadio_region_presets_tag; + getRegionPresetMap(fromRadioScratch.region_presets); state = STATE_SEND_CHANNELS; + config_state = 0; // STATE_SEND_CHANNELS indexes channels starting at 0 break; case STATE_SEND_CHANNELS: @@ -1517,6 +1532,7 @@ bool PhoneAPI::available() case STATE_SEND_CONFIG: case STATE_SEND_MODULECONFIG: case STATE_SEND_METADATA: + case STATE_SEND_REGION_PRESETS: case STATE_SEND_OWN_NODEINFO: case STATE_SEND_FILEMANIFEST: case STATE_SEND_COMPLETE_ID: diff --git a/src/mesh/PhoneAPI.h b/src/mesh/PhoneAPI.h index cd5939b0c..ec7a593c1 100644 --- a/src/mesh/PhoneAPI.h +++ b/src/mesh/PhoneAPI.h @@ -46,6 +46,7 @@ class PhoneAPI STATE_SEND_MY_INFO, // send our my info record STATE_SEND_OWN_NODEINFO, STATE_SEND_METADATA, + STATE_SEND_REGION_PRESETS, // Send the region->valid-preset map (one message) STATE_SEND_CHANNELS, // Send all channels STATE_SEND_CONFIG, // Replacement for the old Radioconfig STATE_SEND_MODULECONFIG, // Send Module specific config diff --git a/src/mesh/RadioInterface.cpp b/src/mesh/RadioInterface.cpp index 0d973e2e2..281587d44 100644 --- a/src/mesh/RadioInterface.cpp +++ b/src/mesh/RadioInterface.cpp @@ -605,6 +605,62 @@ const RegionInfo *getRegion(meshtastic_Config_LoRaConfig_RegionCode code) return r; } +void getRegionPresetMap(meshtastic_LoRaRegionPresetMap &map) +{ + map = meshtastic_LoRaRegionPresetMap_init_zero; + + const size_t maxGroups = sizeof(map.groups) / sizeof(map.groups[0]); + const size_t maxRegions = sizeof(map.region_groups) / sizeof(map.region_groups[0]); + const size_t maxPresets = sizeof(map.groups[0].presets) / sizeof(map.groups[0].presets[0]); + + // Coalesce regions that share an identical preset list into one group. Two + // regions belong to the same group when they share the same RegionProfile + // (which owns the preset list + licensing) AND the same default preset. + // Keyed by profile pointer, not the preset-array pointer: PROFILE_NARROW and + // PROFILE_HAM_100KHZ share PRESETS_NARROW but differ in licensedOnly. + const RegionProfile *groupProfile[sizeof(map.groups) / sizeof(map.groups[0])] = {}; + + for (const RegionInfo *r = regions; r->code != meshtastic_Config_LoRaConfig_RegionCode_UNSET; r++) { + // No room left to map any further region; once full we can't add more, so + // log once and stop. An incomplete map means clients won't constrain the + // omitted regions, so this must be discoverable rather than silent. + if (map.region_groups_count >= maxRegions) { + LOG_ERROR("Region preset map full at %u regions; remaining regions omitted", (unsigned)maxRegions); + break; + } + + // Find the group this region belongs to, or create it. + int gi = -1; + for (pb_size_t g = 0; g < map.groups_count; g++) { + if (groupProfile[g] == r->profile && map.groups[g].default_preset == r->getDefaultPreset()) { + gi = g; + break; + } + } + if (gi < 0) { + if (map.groups_count >= maxGroups) { + // Out of group slots (should not happen for the current table). The + // region can't be advertised; skip it but make the gap visible. + LOG_ERROR("Region preset map out of group slots (%u); region %d omitted", (unsigned)maxGroups, r->code); + continue; + } + gi = map.groups_count++; + groupProfile[gi] = r->profile; + meshtastic_LoRaPresetGroup &grp = map.groups[gi]; + grp.default_preset = r->getDefaultPreset(); + grp.licensed_only = r->profile->licensedOnly; + grp.presets_count = 0; + for (size_t i = 0; r->profile->presets[i] != MODEM_PRESET_END && grp.presets_count < maxPresets; i++) + grp.presets[grp.presets_count++] = r->profile->presets[i]; + } + + // Map this region to its group (capacity checked at the top of the loop). + meshtastic_LoRaRegionPresets &rg = map.region_groups[map.region_groups_count++]; + rg.region = r->code; + rg.group_index = (uint8_t)gi; + } +} + /** * Get duty cycle for current region. EU_866: 10% for routers, 2.5% for mobile. */ diff --git a/src/mesh/generated/meshtastic/mesh.pb.cpp b/src/mesh/generated/meshtastic/mesh.pb.cpp index a68ffabac..ac991c580 100644 --- a/src/mesh/generated/meshtastic/mesh.pb.cpp +++ b/src/mesh/generated/meshtastic/mesh.pb.cpp @@ -96,6 +96,15 @@ PB_BIND(meshtastic_Neighbor, meshtastic_Neighbor, AUTO) PB_BIND(meshtastic_DeviceMetadata, meshtastic_DeviceMetadata, AUTO) +PB_BIND(meshtastic_LoRaPresetGroup, meshtastic_LoRaPresetGroup, AUTO) + + +PB_BIND(meshtastic_LoRaRegionPresets, meshtastic_LoRaRegionPresets, AUTO) + + +PB_BIND(meshtastic_LoRaRegionPresetMap, meshtastic_LoRaRegionPresetMap, 2) + + PB_BIND(meshtastic_Heartbeat, meshtastic_Heartbeat, AUTO) diff --git a/src/mesh/generated/meshtastic/mesh.pb.h b/src/mesh/generated/meshtastic/mesh.pb.h index 5b27f01d1..ce92d81f1 100644 --- a/src/mesh/generated/meshtastic/mesh.pb.h +++ b/src/mesh/generated/meshtastic/mesh.pb.h @@ -1355,6 +1355,53 @@ typedef struct _meshtastic_DeviceMetadata { uint32_t excluded_modules; } meshtastic_DeviceMetadata; +/* A distinct set of legal modem presets shared by one or more LoRa regions. + Regions that have an identical preset list / default / licensing reference + the same group (by index) via LoRaRegionPresetMap.region_groups. This keeps + the whole map small enough to fit in a single FromRadio packet, since most + regions share the one standard preset list. */ +typedef struct _meshtastic_LoRaPresetGroup { + /* The modem presets that are legal for every region referencing this group. */ + pb_size_t presets_count; + meshtastic_Config_LoRaConfig_ModemPreset presets[11]; + /* The firmware's default modem preset for regions in this group. + Always one of `presets`. Clients should select this when switching to one + of these regions, or when the current preset is not legal in the new region. */ + meshtastic_Config_LoRaConfig_ModemPreset default_preset; + /* True if regions referencing this group are for licensed operators only + (e.g. amateur / ham radio bands). Clients should warn or gate accordingly. */ + bool licensed_only; +} meshtastic_LoRaPresetGroup; + +/* Associates a single LoRa region with its preset group. */ +typedef struct _meshtastic_LoRaRegionPresets { + /* The LoRa region this entry describes. */ + meshtastic_Config_LoRaConfig_RegionCode region; + /* Index into LoRaRegionPresetMap.groups for the preset list that is legal + in `region`. */ + uint8_t group_index; +} meshtastic_LoRaRegionPresets; + +/* Map describing which modem presets are valid for each LoRa region. Sent by + the firmware during the want_config handshake (as FromRadio.region_presets) + so that client UIs can prevent illegal region+preset selections. + + Delivery is grouped to save space: `groups` holds each distinct preset list, + and `region_groups` maps every known region to one of those groups by index. + A region that does NOT appear in `region_groups` carries no constraint + information and should not be restricted by the client (e.g. firmware that + predates this message, or a region with no firmware table entry). Clients + must also tolerate this whole message being absent. */ +typedef struct _meshtastic_LoRaRegionPresetMap { + /* One entry per distinct (preset-list, default, licensing) combination. + Referenced by index from `region_groups`. */ + pb_size_t groups_count; + meshtastic_LoRaPresetGroup groups[8]; + /* One entry per known LoRa region, pointing at its preset group. */ + pb_size_t region_groups_count; + meshtastic_LoRaRegionPresets region_groups[38]; +} meshtastic_LoRaRegionPresetMap; + /* Packets from the radio to the phone will appear on the fromRadio characteristic. It will support READ and NOTIFY. When a new packet arrives the device will BLE notify? It will sit in that descriptor until consumed by the phone, @@ -1411,6 +1458,12 @@ typedef struct _meshtastic_FromRadio { to report success or failure. Replaces the earlier scheme of encoding state as magic-string prefixes inside ClientNotification. */ meshtastic_LockdownStatus lockdown_status; + /* Map of which modem presets are legal in each LoRa region. Sent once + during the want_config handshake (right after `metadata`, before the + first `channel`) so client UIs can prevent the user from selecting an + illegal region+preset combination. A region that does not appear in + any group carries no constraint info and should not be restricted. */ + meshtastic_LoRaRegionPresetMap region_presets; }; } meshtastic_FromRadio; @@ -1604,6 +1657,12 @@ extern "C" { #define meshtastic_DeviceMetadata_role_ENUMTYPE meshtastic_Config_DeviceConfig_Role #define meshtastic_DeviceMetadata_hw_model_ENUMTYPE meshtastic_HardwareModel +#define meshtastic_LoRaPresetGroup_presets_ENUMTYPE meshtastic_Config_LoRaConfig_ModemPreset +#define meshtastic_LoRaPresetGroup_default_preset_ENUMTYPE meshtastic_Config_LoRaConfig_ModemPreset + +#define meshtastic_LoRaRegionPresets_region_ENUMTYPE meshtastic_Config_LoRaConfig_RegionCode + + @@ -1641,6 +1700,9 @@ extern "C" { #define meshtastic_NeighborInfo_init_default {0, 0, 0, 0, {meshtastic_Neighbor_init_default, meshtastic_Neighbor_init_default, meshtastic_Neighbor_init_default, meshtastic_Neighbor_init_default, meshtastic_Neighbor_init_default, meshtastic_Neighbor_init_default, meshtastic_Neighbor_init_default, meshtastic_Neighbor_init_default, meshtastic_Neighbor_init_default, meshtastic_Neighbor_init_default}} #define meshtastic_Neighbor_init_default {0, 0, 0, 0} #define meshtastic_DeviceMetadata_init_default {"", 0, 0, 0, 0, 0, _meshtastic_Config_DeviceConfig_Role_MIN, 0, _meshtastic_HardwareModel_MIN, 0, 0, 0} +#define meshtastic_LoRaPresetGroup_init_default {0, {_meshtastic_Config_LoRaConfig_ModemPreset_MIN, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, _meshtastic_Config_LoRaConfig_ModemPreset_MIN}, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, 0} +#define meshtastic_LoRaRegionPresets_init_default {_meshtastic_Config_LoRaConfig_RegionCode_MIN, 0} +#define meshtastic_LoRaRegionPresetMap_init_default {0, {meshtastic_LoRaPresetGroup_init_default, meshtastic_LoRaPresetGroup_init_default, meshtastic_LoRaPresetGroup_init_default, meshtastic_LoRaPresetGroup_init_default, meshtastic_LoRaPresetGroup_init_default, meshtastic_LoRaPresetGroup_init_default, meshtastic_LoRaPresetGroup_init_default, meshtastic_LoRaPresetGroup_init_default}, 0, {meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default, meshtastic_LoRaRegionPresets_init_default}} #define meshtastic_Heartbeat_init_default {0} #define meshtastic_NodeRemoteHardwarePin_init_default {0, false, meshtastic_RemoteHardwarePin_init_default} #define meshtastic_ChunkedPayload_init_default {0, 0, 0, {0, {0}}} @@ -1676,6 +1738,9 @@ extern "C" { #define meshtastic_NeighborInfo_init_zero {0, 0, 0, 0, {meshtastic_Neighbor_init_zero, meshtastic_Neighbor_init_zero, meshtastic_Neighbor_init_zero, meshtastic_Neighbor_init_zero, meshtastic_Neighbor_init_zero, meshtastic_Neighbor_init_zero, meshtastic_Neighbor_init_zero, meshtastic_Neighbor_init_zero, meshtastic_Neighbor_init_zero, meshtastic_Neighbor_init_zero}} #define meshtastic_Neighbor_init_zero {0, 0, 0, 0} #define meshtastic_DeviceMetadata_init_zero {"", 0, 0, 0, 0, 0, _meshtastic_Config_DeviceConfig_Role_MIN, 0, _meshtastic_HardwareModel_MIN, 0, 0, 0} +#define meshtastic_LoRaPresetGroup_init_zero {0, {_meshtastic_Config_LoRaConfig_ModemPreset_MIN, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, _meshtastic_Config_LoRaConfig_ModemPreset_MIN}, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, 0} +#define meshtastic_LoRaRegionPresets_init_zero {_meshtastic_Config_LoRaConfig_RegionCode_MIN, 0} +#define meshtastic_LoRaRegionPresetMap_init_zero {0, {meshtastic_LoRaPresetGroup_init_zero, meshtastic_LoRaPresetGroup_init_zero, meshtastic_LoRaPresetGroup_init_zero, meshtastic_LoRaPresetGroup_init_zero, meshtastic_LoRaPresetGroup_init_zero, meshtastic_LoRaPresetGroup_init_zero, meshtastic_LoRaPresetGroup_init_zero, meshtastic_LoRaPresetGroup_init_zero}, 0, {meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero, meshtastic_LoRaRegionPresets_init_zero}} #define meshtastic_Heartbeat_init_zero {0} #define meshtastic_NodeRemoteHardwarePin_init_zero {0, false, meshtastic_RemoteHardwarePin_init_zero} #define meshtastic_ChunkedPayload_init_zero {0, 0, 0, {0, {0}}} @@ -1866,6 +1931,13 @@ extern "C" { #define meshtastic_DeviceMetadata_hasRemoteHardware_tag 10 #define meshtastic_DeviceMetadata_hasPKC_tag 11 #define meshtastic_DeviceMetadata_excluded_modules_tag 12 +#define meshtastic_LoRaPresetGroup_presets_tag 1 +#define meshtastic_LoRaPresetGroup_default_preset_tag 2 +#define meshtastic_LoRaPresetGroup_licensed_only_tag 3 +#define meshtastic_LoRaRegionPresets_region_tag 1 +#define meshtastic_LoRaRegionPresets_group_index_tag 2 +#define meshtastic_LoRaRegionPresetMap_groups_tag 1 +#define meshtastic_LoRaRegionPresetMap_region_groups_tag 2 #define meshtastic_FromRadio_id_tag 1 #define meshtastic_FromRadio_packet_tag 2 #define meshtastic_FromRadio_my_info_tag 3 @@ -1884,6 +1956,7 @@ extern "C" { #define meshtastic_FromRadio_clientNotification_tag 16 #define meshtastic_FromRadio_deviceuiConfig_tag 17 #define meshtastic_FromRadio_lockdown_status_tag 18 +#define meshtastic_FromRadio_region_presets_tag 19 #define meshtastic_Heartbeat_nonce_tag 1 #define meshtastic_ToRadio_packet_tag 1 #define meshtastic_ToRadio_want_config_id_tag 3 @@ -2128,7 +2201,8 @@ X(a, STATIC, ONEOF, MESSAGE, (payload_variant,mqttClientProxyMessage,mqttC X(a, STATIC, ONEOF, MESSAGE, (payload_variant,fileInfo,fileInfo), 15) \ X(a, STATIC, ONEOF, MESSAGE, (payload_variant,clientNotification,clientNotification), 16) \ X(a, STATIC, ONEOF, MESSAGE, (payload_variant,deviceuiConfig,deviceuiConfig), 17) \ -X(a, STATIC, ONEOF, MESSAGE, (payload_variant,lockdown_status,lockdown_status), 18) +X(a, STATIC, ONEOF, MESSAGE, (payload_variant,lockdown_status,lockdown_status), 18) \ +X(a, STATIC, ONEOF, MESSAGE, (payload_variant,region_presets,region_presets), 19) #define meshtastic_FromRadio_CALLBACK NULL #define meshtastic_FromRadio_DEFAULT NULL #define meshtastic_FromRadio_payload_variant_packet_MSGTYPE meshtastic_MeshPacket @@ -2146,6 +2220,7 @@ X(a, STATIC, ONEOF, MESSAGE, (payload_variant,lockdown_status,lockdown_sta #define meshtastic_FromRadio_payload_variant_clientNotification_MSGTYPE meshtastic_ClientNotification #define meshtastic_FromRadio_payload_variant_deviceuiConfig_MSGTYPE meshtastic_DeviceUIConfig #define meshtastic_FromRadio_payload_variant_lockdown_status_MSGTYPE meshtastic_LockdownStatus +#define meshtastic_FromRadio_payload_variant_region_presets_MSGTYPE meshtastic_LoRaRegionPresetMap #define meshtastic_LockdownStatus_FIELDLIST(X, a) \ X(a, STATIC, SINGULAR, UENUM, state, 1) \ @@ -2264,6 +2339,27 @@ X(a, STATIC, SINGULAR, UINT32, excluded_modules, 12) #define meshtastic_DeviceMetadata_CALLBACK NULL #define meshtastic_DeviceMetadata_DEFAULT NULL +#define meshtastic_LoRaPresetGroup_FIELDLIST(X, a) \ +X(a, STATIC, REPEATED, UENUM, presets, 1) \ +X(a, STATIC, SINGULAR, UENUM, default_preset, 2) \ +X(a, STATIC, SINGULAR, BOOL, licensed_only, 3) +#define meshtastic_LoRaPresetGroup_CALLBACK NULL +#define meshtastic_LoRaPresetGroup_DEFAULT NULL + +#define meshtastic_LoRaRegionPresets_FIELDLIST(X, a) \ +X(a, STATIC, SINGULAR, UENUM, region, 1) \ +X(a, STATIC, SINGULAR, UINT32, group_index, 2) +#define meshtastic_LoRaRegionPresets_CALLBACK NULL +#define meshtastic_LoRaRegionPresets_DEFAULT NULL + +#define meshtastic_LoRaRegionPresetMap_FIELDLIST(X, a) \ +X(a, STATIC, REPEATED, MESSAGE, groups, 1) \ +X(a, STATIC, REPEATED, MESSAGE, region_groups, 2) +#define meshtastic_LoRaRegionPresetMap_CALLBACK NULL +#define meshtastic_LoRaRegionPresetMap_DEFAULT NULL +#define meshtastic_LoRaRegionPresetMap_groups_MSGTYPE meshtastic_LoRaPresetGroup +#define meshtastic_LoRaRegionPresetMap_region_groups_MSGTYPE meshtastic_LoRaRegionPresets + #define meshtastic_Heartbeat_FIELDLIST(X, a) \ X(a, STATIC, SINGULAR, UINT32, nonce, 1) #define meshtastic_Heartbeat_CALLBACK NULL @@ -2328,6 +2424,9 @@ extern const pb_msgdesc_t meshtastic_Compressed_msg; extern const pb_msgdesc_t meshtastic_NeighborInfo_msg; extern const pb_msgdesc_t meshtastic_Neighbor_msg; extern const pb_msgdesc_t meshtastic_DeviceMetadata_msg; +extern const pb_msgdesc_t meshtastic_LoRaPresetGroup_msg; +extern const pb_msgdesc_t meshtastic_LoRaRegionPresets_msg; +extern const pb_msgdesc_t meshtastic_LoRaRegionPresetMap_msg; extern const pb_msgdesc_t meshtastic_Heartbeat_msg; extern const pb_msgdesc_t meshtastic_NodeRemoteHardwarePin_msg; extern const pb_msgdesc_t meshtastic_ChunkedPayload_msg; @@ -2365,6 +2464,9 @@ extern const pb_msgdesc_t meshtastic_ChunkedPayloadResponse_msg; #define meshtastic_NeighborInfo_fields &meshtastic_NeighborInfo_msg #define meshtastic_Neighbor_fields &meshtastic_Neighbor_msg #define meshtastic_DeviceMetadata_fields &meshtastic_DeviceMetadata_msg +#define meshtastic_LoRaPresetGroup_fields &meshtastic_LoRaPresetGroup_msg +#define meshtastic_LoRaRegionPresets_fields &meshtastic_LoRaRegionPresets_msg +#define meshtastic_LoRaRegionPresetMap_fields &meshtastic_LoRaRegionPresetMap_msg #define meshtastic_Heartbeat_fields &meshtastic_Heartbeat_msg #define meshtastic_NodeRemoteHardwarePin_fields &meshtastic_NodeRemoteHardwarePin_msg #define meshtastic_ChunkedPayload_fields &meshtastic_ChunkedPayload_msg @@ -2388,6 +2490,9 @@ extern const pb_msgdesc_t meshtastic_ChunkedPayloadResponse_msg; #define meshtastic_KeyVerificationNumberInform_size 58 #define meshtastic_KeyVerificationNumberRequest_size 52 #define meshtastic_KeyVerification_size 79 +#define meshtastic_LoRaPresetGroup_size 26 +#define meshtastic_LoRaRegionPresetMap_size 490 +#define meshtastic_LoRaRegionPresets_size 5 #define meshtastic_LockdownStatus_size 53 #define meshtastic_LogRecord_size 426 #define meshtastic_LowEntropyKey_size 0 diff --git a/test/test_radio/test_main.cpp b/test/test_radio/test_main.cpp index 104e6b36b..892c7f695 100644 --- a/test/test_radio/test_main.cpp +++ b/test/test_radio/test_main.cpp @@ -200,6 +200,87 @@ static void test_applyModemConfig_customCodingRateLowerThanPreset() TEST_ASSERT_EQUAL_UINT8(8, testRadio->getCr()); } +// ----------------------------------------------------------------------- +// getRegionPresetMap() — region->valid-preset map sent to clients during want_config +// ----------------------------------------------------------------------- + +static size_t countKnownRegions() +{ + size_t n = 0; + for (const RegionInfo *r = regions; r->code != meshtastic_Config_LoRaConfig_RegionCode_UNSET; r++) + n++; + return n; +} + +// Every region in the firmware table (except the UNSET sentinel) must appear +// exactly once in the map, and all counts must stay within the mesh.options bounds +// (exceeding them would mean nanopb silently truncates the wire message). +static void test_regionPresetMap_coversAllRegionsWithinBounds() +{ + meshtastic_LoRaRegionPresetMap map; + getRegionPresetMap(map); + + const size_t known = countKnownRegions(); + TEST_ASSERT_EQUAL_UINT((unsigned)known, (unsigned)map.region_groups_count); + + // Bounds derived from the generated nanopb arrays (mesh.options max_count), so + // this stays correct if those bounds change. + const size_t maxGroups = sizeof(map.groups) / sizeof(map.groups[0]); + const size_t maxRegions = sizeof(map.region_groups) / sizeof(map.region_groups[0]); + TEST_ASSERT_GREATER_THAN_UINT(0, map.groups_count); + TEST_ASSERT_LESS_OR_EQUAL_UINT((unsigned)maxGroups, map.groups_count); + TEST_ASSERT_LESS_OR_EQUAL_UINT((unsigned)maxRegions, map.region_groups_count); + + // Each known region appears exactly once. + for (const RegionInfo *r = regions; r->code != meshtastic_Config_LoRaConfig_RegionCode_UNSET; r++) { + int hits = 0; + for (pb_size_t i = 0; i < map.region_groups_count; i++) + if (map.region_groups[i].region == r->code) + hits++; + TEST_ASSERT_EQUAL_INT(1, hits); + } +} + +// The advertised presets must agree with the live region table: every preset is +// legal in its region, the default is among them, and the licensed flag matches. +static void test_regionPresetMap_matchesRegionTable() +{ + meshtastic_LoRaRegionPresetMap map; + getRegionPresetMap(map); + + for (pb_size_t i = 0; i < map.region_groups_count; i++) { + meshtastic_Config_LoRaConfig_RegionCode code = map.region_groups[i].region; + uint8_t gi = map.region_groups[i].group_index; + TEST_ASSERT_LESS_THAN_UINT(map.groups_count, gi); + + const meshtastic_LoRaPresetGroup &grp = map.groups[gi]; + const RegionInfo *r = getRegion(code); + + // Group's list is non-empty, within the generated array bound, and is the + // region's full list. + const size_t maxPresets = sizeof(grp.presets) / sizeof(grp.presets[0]); + TEST_ASSERT_GREATER_THAN_UINT(0, grp.presets_count); + TEST_ASSERT_LESS_OR_EQUAL_UINT((unsigned)maxPresets, grp.presets_count); + TEST_ASSERT_EQUAL_UINT((unsigned)r->getNumPresets(), (unsigned)grp.presets_count); + + // Every advertised preset is legal in this region. + for (pb_size_t p = 0; p < grp.presets_count; p++) + TEST_ASSERT_TRUE(r->supportsPreset(grp.presets[p])); + + // Default preset matches the table, is legal, and is present in the list. + TEST_ASSERT_EQUAL(r->getDefaultPreset(), grp.default_preset); + TEST_ASSERT_TRUE(r->supportsPreset(grp.default_preset)); + bool defaultInList = false; + for (pb_size_t p = 0; p < grp.presets_count; p++) + if (grp.presets[p] == grp.default_preset) + defaultInList = true; + TEST_ASSERT_TRUE(defaultInList); + + // Licensed flag matches the region's profile. + TEST_ASSERT_EQUAL(r->profile->licensedOnly, grp.licensed_only); + } +} + void setUp(void) { mockMeshService = new MockMeshService(); @@ -241,6 +322,8 @@ void setup() RUN_TEST(test_applyModemConfig_codingRateMatchesPreset); RUN_TEST(test_applyModemConfig_customCodingRateHigherThanPreset); RUN_TEST(test_applyModemConfig_customCodingRateLowerThanPreset); + RUN_TEST(test_regionPresetMap_coversAllRegionsWithinBounds); + RUN_TEST(test_regionPresetMap_matchesRegionTable); exit(UNITY_END()); } From 161cd265190a47f467041706ad243f3e9367114f Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Fri, 19 Jun 2026 20:49:48 -0500 Subject: [PATCH 20/86] Add conditional compilation for warm node count checks --- src/mesh/NodeDB.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index 7e5625940..98d6a3f32 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -1629,6 +1629,7 @@ bool NodeDB::enforceSatelliteCaps() return trimmedAny; } +#if WARM_NODE_COUNT > 0 // Classify an evicted node's hop-protected category for the warm tier. Favorite/ignored/ // verified are local flags (rarely reach warm — they're eviction-protected — but classify // them if they do); otherwise tracker/sensor/tak_tracker are role-protected. @@ -1647,6 +1648,7 @@ static uint8_t warmProtectedCategory(const meshtastic_NodeInfoLite &n) // loudly if a new role outgrows it, rather than silently truncating role on eviction. static_assert(_meshtastic_Config_DeviceConfig_Role_MAX <= WARM_ROLE_MASK, "device role no longer fits the 4-bit warm metadata field"); +#endif // WARM_NODE_COUNT > 0 void NodeDB::cleanupMeshDB() { From d8d92b7b71eab51d1c2e88f0251eabf286061709 Mon Sep 17 00:00:00 2001 From: Tom <116762865+NomDeTom@users.noreply.github.com> Date: Sat, 20 Jun 2026 23:08:29 +0100 Subject: [PATCH 21/86] Fix/zerohop and hopscale mqtt (#10753) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix for prehopdrop on zerohop packets * hopscaling: exclude MQTT-origin packets from the node histogram The hop-scaling histogram gate only checked transport_mechanism == TRANSPORT_LORA, which excludes packets received from the broker but NOT MQTT-origin packets that a gateway rebroadcasts onto LoRa (those arrive as TRANSPORT_LORA with via_mqtt set). Counting them inflates the local mesh-size/density estimate with nodes that aren't real RF participants — and since the bridged copy usually carries hop_start==0 they land in the hop-0 bucket, the one that pulls getLastRequiredHop() lowest, over- shrinking NodeInfo/position/telemetry hop limits. Add `!mp.via_mqtt` to the gate. Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Claude Sonnet 4.6 --- src/mesh/NodeDB.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index 98d6a3f32..5653011fa 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -2821,6 +2821,10 @@ HopStartStatus classifyHopStart(const meshtastic_MeshPacket &p) return HopStartStatus::INVALID; if (p.hop_start == 0) { + // hop_start == hop_limit == 0: intentional zero-hop broadcast (e.g. beacon). Valid by definition — + // the packet was never meant to travel any hops, so no hop_start ambiguity applies. + if (p.hop_limit == 0) + return HopStartStatus::VALID; // Firmware prior to 2.3.0 (585805c) lacked a hop_start field. Firmware version 2.5.0 (bf34329) introduced a // bitfield that is always present. Use the presence of the bitfield to determine if the origin's firmware // version is guaranteed to have hop_start populated. Note that this can only be done for decoded packets as @@ -3161,8 +3165,14 @@ void NodeDB::updateFrom(const meshtastic_MeshPacket &mp) mp.via_mqtt); // Store if we received this packet via MQTT #if HAS_VARIABLE_HOPS - // Only sample packets that arrived over LoRa. - if (mp.transport_mechanism == meshtastic_MeshPacket_TransportMechanism_TRANSPORT_LORA && hopScalingModule) { + // Only sample genuine RF-origin packets. The transport check excludes packets received + // directly from the broker (TRANSPORT_MQTT), but an MQTT-origin packet rebroadcast onto + // LoRa by a gateway arrives as TRANSPORT_LORA with via_mqtt set — count those would + // inflate the local mesh-size estimate with non-RF nodes (and they usually carry + // hop_start==0, landing in the hop-0 bucket that pulls the recommendation lowest), so + // exclude via_mqtt too. + if (mp.transport_mechanism == meshtastic_MeshPacket_TransportMechanism_TRANSPORT_LORA && !mp.via_mqtt && + hopScalingModule) { uint8_t hopCount = std::max(int8_t(0), getHopsAway(mp)); hopScalingModule->samplePacketForHistogram(mp.from, hopCount); } From c51c01607d8972a4e8a228c6c45edb5e0606dac2 Mon Sep 17 00:00:00 2001 From: Tom <116762865+NomDeTom@users.noreply.github.com> Date: Sun, 21 Jun 2026 02:35:14 +0100 Subject: [PATCH 22/86] Traffic Management module: dedup, rate limiting, role-aware policing (#10706) Adds the Traffic Management module (TMM) plus the NodeDB/warm-store and next-hop foundations it builds on: - Unified per-node cache (flat array, 8-bit relative ticks) shared by all features; role-aware throttles for tracker / lost-and-found. - Position deduplication: drop unchanged position rebroadcasts within a configurable interval; precision driven off the channel ceiling (clamped to the public-key max on well-known channels). Enabled by default at 11h. - Per-node rate limiting and unknown-packet filtering (config-driven; a non-zero companion field enables each feature -- no bool toggles). - NodeInfo direct response from cache with role-based hop clamps. - Persistent next-hop overflow store: confirmed hops have no TTL, are seeded from NodeInfoLite at boot, and survive hot-store eviction. - Three-tier sender-role resolution (hot NodeInfoLite -> warm store -> TMM cache). Role is cached write-time (seeded on first track, refreshed from NodeInfo), pins its cache entry like a next-hop hint, and is evicted last. - Warm store caches device role + protected category across reboot/eviction. - PositionModule stationary floor for tracker / lost-and-found. - PSRAM gating for warm/satellite/TMM cache sizes; STM32WL excluded. Protobufs: TrafficManagementConfig trimmed to the five uint32 fields actually used; submodule repointed to protobufs develop. Co-authored-by: Claude Sonnet 4.6 --- protobufs | 2 +- src/mesh/Channels.cpp | 18 + src/mesh/Channels.h | 6 + src/mesh/Default.h | 13 +- src/mesh/NodeDB.cpp | 24 +- src/mesh/NodeDB.h | 5 + src/mesh/PositionPrecision.cpp | 10 +- src/mesh/PositionPrecision.h | 6 + src/mesh/Router.cpp | 19 +- src/mesh/generated/meshtastic/deviceonly.pb.h | 2 +- src/mesh/generated/meshtastic/localonly.pb.h | 2 +- .../generated/meshtastic/module_config.pb.h | 61 +- src/mesh/mesh-pb-constants.h | 42 +- src/modules/AdminModule.cpp | 4 + src/modules/Modules.cpp | 3 +- src/modules/PositionModule.cpp | 54 +- src/modules/PositionModule.h | 14 + src/modules/TrafficManagementModule.cpp | 550 ++++++++------- src/modules/TrafficManagementModule.h | 230 +++---- test/test_position_module/test_main.cpp | 81 +++ test/test_traffic_management/test_main.cpp | 637 ++++++++++++++---- userPrefs.jsonc | 1 + variants/esp32s3/heltec_v4/variant.h | 3 - variants/esp32s3/heltec_v4_r8/variant.h | 3 - variants/esp32s3/station-g2/variant.h | 3 - variants/native/portduino/variant.h | 3 - 26 files changed, 1186 insertions(+), 610 deletions(-) create mode 100644 test/test_position_module/test_main.cpp diff --git a/protobufs b/protobufs index f16a13e43..03314e639 160000 --- a/protobufs +++ b/protobufs @@ -1 +1 @@ -Subproject commit f16a13e43393f3c6a88a1da36e75dfc29f448d3e +Subproject commit 03314e63950bda910695eaeb4ba8169dae0425ef diff --git a/src/mesh/Channels.cpp b/src/mesh/Channels.cpp index 457e64461..5b5875971 100644 --- a/src/mesh/Channels.cpp +++ b/src/mesh/Channels.cpp @@ -440,6 +440,24 @@ bool Channels::usesPublicKey(ChannelIndex chIndex) return (psk.size == sizeof(defaultpsk) && memcmp(psk.bytes, defaultpsk, sizeof(defaultpsk) - 1) == 0); } +bool Channels::isWellKnownChannel(ChannelIndex chIndex) +{ + const auto &ch = getByIndex(chIndex); + // Absent (unencrypted) or single-byte PSK — all the well-known key indexes + if (ch.settings.psk.size > 1) + return false; + + const char *name = getName(chIndex); + for (int p = _meshtastic_Config_LoRaConfig_ModemPreset_MIN; p <= _meshtastic_Config_LoRaConfig_ModemPreset_MAX; p++) { + const char *presetName = + DisplayFormatters::getModemPresetDisplayName(static_cast(p), false, true); + // Presets without a display name fall through to "Invalid" — never a match + if (strcmp(presetName, "Invalid") != 0 && strcmp(name, presetName) == 0) + return true; + } + return false; +} + bool Channels::hasDefaultChannel() { // If we don't use a preset or the default frequency slot, or we override the frequency, we don't have a default channel diff --git a/src/mesh/Channels.h b/src/mesh/Channels.h index 9ba84e2ee..7392284f5 100644 --- a/src/mesh/Channels.h +++ b/src/mesh/Channels.h @@ -88,6 +88,12 @@ class Channels // Returns true if this channel's effective key is publicly decryptable (open or well-known/default PSK). bool usesPublicKey(ChannelIndex chIndex); + // Returns true if the channel is "well known": its PSK is absent or a + // single-byte well-known key index, AND its name is any modem-preset + // display name (e.g. a channel named "LongFast" counts even while the + // radio runs MediumFast). Broader than isDefaultChannel, which only + // matches the current preset's name and PSK byte 1. + bool isWellKnownChannel(ChannelIndex chIndex); // Returns true if we can be reached via a channel with the default settings given a region and modem preset bool hasDefaultChannel(); diff --git a/src/mesh/Default.h b/src/mesh/Default.h index d638ca2a9..e802a0324 100644 --- a/src/mesh/Default.h +++ b/src/mesh/Default.h @@ -18,6 +18,9 @@ #define default_telemetry_broadcast_interval_secs IF_ROUTER(ONE_DAY / 2, 60 * 60) #define default_broadcast_interval_secs IF_ROUTER(ONE_DAY / 2, 60 * 60) #define default_broadcast_smart_minimum_interval_secs 5 * 60 +// Floor for our own position broadcasts when stationary (unchanged beyond the broadcast +// precision) or fixed_position: identical positions get deduped by traffic management anyway. +#define default_position_stationary_broadcast_secs (12 * 60 * 60) #define min_default_broadcast_interval_secs IF_ROUTER(ONE_DAY / 2, 60 * 60) #define min_default_broadcast_smart_minimum_interval_secs 5 * 60 #define default_wait_bluetooth_secs IF_ROUTER(1, 60) @@ -34,8 +37,14 @@ enum class TrafficType { POSITION, TELEMETRY }; // Traffic management defaults -#define default_traffic_mgmt_position_precision_bits 19 // ~90m grid cells (±45m) -#define default_traffic_mgmt_position_min_interval_secs (ONE_DAY / 2) // 12 hours between identical positions +#define default_traffic_mgmt_position_precision_bits 19 // ~90m grid cells (±45m) +#define default_traffic_mgmt_position_min_interval_secs (11 * 60 * 60) // 11 hours between identical positions +// Role cap: tracker-role origins may refresh a duplicate position this often (vs the 11h default). +#define default_traffic_mgmt_tracker_position_min_interval_secs (60 * 60) // 1 hour +// Role cap: lost-and-found origins may refresh a duplicate position this often, so a lost +// device updates frequently without flooding. (Quantised to the dedup tick: ~2 ticks.) +// Unlike before, lost-and-found is NOT exempt from the relayed precision clamp. +#define default_traffic_mgmt_lost_and_found_position_min_interval_secs (15 * 60) // 15 minutes // Hop scaling defaults #define default_hop_scaling_min_target_nodes 40 // walk threshold: first hop reaching this cumulative count diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index 5653011fa..27c498e97 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -1155,8 +1155,12 @@ static void installTrafficManagementDefaults(meshtastic_LocalModuleConfig &mc) { mc.has_traffic_management = true; mc.traffic_management = meshtastic_ModuleConfig_TrafficManagementConfig_init_zero; - mc.traffic_management.enabled = true; - mc.traffic_management.position_dedup_enabled = true; +#if HAS_TRAFFIC_MANAGEMENT + // Position dedup ships enabled at the 11-hour default window on all supported targets. + // STM32WL is excluded at compile time (HAS_TRAFFIC_MANAGEMENT=0 in mesh-pb-constants.h). + // Set position_min_interval_secs=0 at runtime to disable dedup. + mc.traffic_management.position_min_interval_secs = default_traffic_mgmt_position_min_interval_secs; +#endif } void NodeDB::installDefaultModuleConfig() @@ -3450,6 +3454,20 @@ bool NodeDB::copyPublicKey(NodeNum n, meshtastic_NodeInfoLite_public_key_t &out) return false; } +meshtastic_Config_DeviceConfig_Role NodeDB::getNodeRole(NodeNum n) +{ + const meshtastic_NodeInfoLite *info = getMeshNode(n); + if (nodeInfoLiteHasUser(info)) + return info->role; +#if WARM_NODE_COUNT > 0 + // Hot-store miss: fall back to the role the warm tier cached at eviction. + uint8_t role = 0, prot = 0; + if (warmStore.lookupMeta(n, role, prot)) + return static_cast(role); +#endif + return meshtastic_Config_DeviceConfig_Role_CLIENT; +} + /// Find a node in our DB, create an empty NodeInfo if missing meshtastic_NodeInfoLite *NodeDB::getOrCreateMeshNode(NodeNum n) { @@ -3672,6 +3690,8 @@ bool NodeDB::createNewIdentity() myNodeInfo.my_node_num = newNodeNum; meshtastic_NodeInfoLite *info = getOrCreateMeshNode(getNodeNum()); + if (!info) + return false; TypeConversions::CopyUserToNodeInfoLite(info, owner); return true; diff --git a/src/mesh/NodeDB.h b/src/mesh/NodeDB.h index fa9c886f7..b7f3d7f9e 100644 --- a/src/mesh/NodeDB.h +++ b/src/mesh/NodeDB.h @@ -341,6 +341,11 @@ class NodeDB /// tier. Returns false if we don't know a key for n. bool copyPublicKey(NodeNum n, meshtastic_NodeInfoLite_public_key_t &out); + /// Resolve a node's device role — hot store (with user) first, then the role + /// cached in the warm tier, else CLIENT. Lets role-aware policy keep firing for + /// nodes that have aged out of the hot store. + meshtastic_Config_DeviceConfig_Role getNodeRole(NodeNum n); + /// last_heard of a hot-store node, or 0 if absent. Plain scan of meshNodes /// with no allocation side effects (unlike getOrCreateMeshNode). uint32_t hotNodeLastHeard(NodeNum n) const; diff --git a/src/mesh/PositionPrecision.cpp b/src/mesh/PositionPrecision.cpp index b5b29903b..4302531a5 100644 --- a/src/mesh/PositionPrecision.cpp +++ b/src/mesh/PositionPrecision.cpp @@ -28,8 +28,11 @@ uint32_t getPositionPrecisionForChannel(uint8_t channelIndex) return precision; } -static int32_t truncateCoordinate(int32_t coordinate, uint32_t precision) +int32_t truncateCoordinate(int32_t coordinate, uint32_t precision) { + if (precision == 0 || precision >= 32) + return coordinate; + uint32_t coordinateBits = static_cast(coordinate); uint32_t truncated = coordinateBits & (UINT32_MAX << (32 - precision)); @@ -39,6 +42,11 @@ static int32_t truncateCoordinate(int32_t coordinate, uint32_t precision) return static_cast(truncated); } +int32_t truncateCoordinate(int32_t coordinate, uint8_t precision) +{ + return truncateCoordinate(coordinate, static_cast(precision)); +} + void applyPositionPrecision(meshtastic_Position &position, uint32_t precision) { if (precision == 0) { diff --git a/src/mesh/PositionPrecision.h b/src/mesh/PositionPrecision.h index 1955cf80f..8299f50a9 100644 --- a/src/mesh/PositionPrecision.h +++ b/src/mesh/PositionPrecision.h @@ -15,6 +15,12 @@ uint32_t getPositionPrecisionForChannel(const meshtastic_Channel &channel); // Configured precision, clamped to MAX_POSITION_PRECISION_PUBLIC_KEY when the channel's effective key is publicly decryptable. uint32_t getPositionPrecisionForChannel(uint8_t channelIndex); + +// Truncate a single latitude_i/longitude_i to `precision` significant bits, centered in the +// resulting grid cell (stable under GPS jitter). precision 0 or >=32 returns the value unchanged. +// The return is the coordinate (int32_t); the uint8_t overload only narrows the precision arg. +int32_t truncateCoordinate(int32_t coordinate, uint32_t precision); +int32_t truncateCoordinate(int32_t coordinate, uint8_t precision); void applyPositionPrecision(meshtastic_Position &position, uint32_t precision); bool applyPositionPrecision(meshtastic_MeshPacket &packet, uint32_t precision); bool applyPositionPrecisionForChannel(meshtastic_MeshPacket &packet, uint8_t channelIndex); diff --git a/src/mesh/Router.cpp b/src/mesh/Router.cpp index 50d0f2bcc..d52fa49f7 100644 --- a/src/mesh/Router.cpp +++ b/src/mesh/Router.cpp @@ -100,19 +100,12 @@ bool Router::shouldDecrementHopLimit(const meshtastic_MeshPacket *p) return true; } -#if HAS_TRAFFIC_MANAGEMENT - // When router_preserve_hops is enabled, preserve hops for decoded packets that are not - // position or telemetry (those have their own exhaust_hop controls). - if (moduleConfig.has_traffic_management && moduleConfig.traffic_management.enabled && - moduleConfig.traffic_management.router_preserve_hops && p->which_payload_variant == meshtastic_MeshPacket_decoded_tag && - p->decoded.portnum != meshtastic_PortNum_POSITION_APP && p->decoded.portnum != meshtastic_PortNum_TELEMETRY_APP) { - LOG_DEBUG("Router hop preserved: port=%d from=0x%08x (traffic_management)", p->decoded.portnum, getFrom(p)); - if (trafficManagementModule) { - trafficManagementModule->recordRouterHopPreserved(); - } - return false; - } -#endif + // router_preserve_hops: not suitable right now — removed from config until + // the right heuristics for when to preserve vs. exhaust hops are established. + // #if HAS_TRAFFIC_MANAGEMENT + // if (moduleConfig.has_traffic_management && + // moduleConfig.traffic_management.router_preserve_hops && ...) { ... } + // #endif // For subsequent hops, preserve hop_limit only when the previous relay is UNAMBIGUOUSLY a favorite // router. The relay_node byte is just the last byte of a 32-bit node number, so on a dense mesh it diff --git a/src/mesh/generated/meshtastic/deviceonly.pb.h b/src/mesh/generated/meshtastic/deviceonly.pb.h index 474b332a8..5bd116730 100644 --- a/src/mesh/generated/meshtastic/deviceonly.pb.h +++ b/src/mesh/generated/meshtastic/deviceonly.pb.h @@ -452,7 +452,7 @@ extern const pb_msgdesc_t meshtastic_BackupPreferences_msg; /* Maximum encoded size of messages (where known) */ /* meshtastic_NodeDatabase_size depends on runtime parameters */ #define MESHTASTIC_MESHTASTIC_DEVICEONLY_PB_H_MAX_SIZE meshtastic_BackupPreferences_size -#define meshtastic_BackupPreferences_size 2432 +#define meshtastic_BackupPreferences_size 2410 #define meshtastic_ChannelFile_size 718 #define meshtastic_DeviceState_size 1944 #define meshtastic_NodeEnvironmentEntry_size 170 diff --git a/src/mesh/generated/meshtastic/localonly.pb.h b/src/mesh/generated/meshtastic/localonly.pb.h index 27f5ad7bf..3cb4a5a57 100644 --- a/src/mesh/generated/meshtastic/localonly.pb.h +++ b/src/mesh/generated/meshtastic/localonly.pb.h @@ -206,7 +206,7 @@ extern const pb_msgdesc_t meshtastic_LocalModuleConfig_msg; /* Maximum encoded size of messages (where known) */ #define MESHTASTIC_MESHTASTIC_LOCALONLY_PB_H_MAX_SIZE meshtastic_LocalModuleConfig_size #define meshtastic_LocalConfig_size 757 -#define meshtastic_LocalModuleConfig_size 820 +#define meshtastic_LocalModuleConfig_size 798 #ifdef __cplusplus } /* extern "C" */ diff --git a/src/mesh/generated/meshtastic/module_config.pb.h b/src/mesh/generated/meshtastic/module_config.pb.h index 25937e972..c7edfa0aa 100644 --- a/src/mesh/generated/meshtastic/module_config.pb.h +++ b/src/mesh/generated/meshtastic/module_config.pb.h @@ -232,34 +232,23 @@ typedef struct _meshtastic_ModuleConfig_PaxcounterConfig { /* Config for the Traffic Management module. Provides packet inspection and traffic shaping to help reduce channel utilization */ typedef struct _meshtastic_ModuleConfig_TrafficManagementConfig { - /* Master enable for traffic management module */ - bool enabled; - /* Enable position deduplication to drop redundant position broadcasts */ - bool position_dedup_enabled; - /* Number of bits of precision for position deduplication (0-32) */ - uint32_t position_precision_bits; - /* Minimum interval in seconds between position updates from the same node */ + /* Minimum interval in seconds between position updates from the same node. + A non-zero value implicitly enables the suppression window; 0 disables it. */ uint32_t position_min_interval_secs; - /* Enable direct response to NodeInfo requests from local cache */ - bool nodeinfo_direct_response; - /* Minimum hop distance from requestor before responding to NodeInfo requests */ + /* Maximum hop distance from the requestor at which direct NodeInfo responses + are served from the local cache. A non-zero value implicitly enables direct + response; 0 disables it. */ uint32_t nodeinfo_direct_response_max_hops; - /* Enable per-node rate limiting to throttle chatty nodes */ - bool rate_limit_enabled; - /* Time window in seconds for rate limiting calculations */ + /* Time window in seconds for per-node rate limiting. + A non-zero value implicitly enables rate limiting; 0 disables it. */ uint32_t rate_limit_window_secs; - /* Maximum packets allowed per node within the rate limit window */ + /* Maximum packets allowed per node within the rate limit window. + A non-zero value implicitly enables rate limiting; 0 disables it. */ uint32_t rate_limit_max_packets; - /* Enable dropping of unknown/undecryptable packets per rate_limit_window_secs */ - bool drop_unknown_enabled; - /* Number of unknown packets before dropping from a node */ + /* Maximum unknown/undecryptable packets per rate window before the source + is dropped. A non-zero value implicitly enables unknown-packet filtering; + 0 disables it. */ uint32_t unknown_packet_threshold; - /* Set hop_limit to 0 for relayed telemetry broadcasts (own packets unaffected) */ - bool exhaust_hop_telemetry; - /* Set hop_limit to 0 for relayed position broadcasts (own packets unaffected) */ - bool exhaust_hop_position; - /* Preserve hop_limit for router-to-router traffic */ - bool router_preserve_hops; } meshtastic_ModuleConfig_TrafficManagementConfig; /* Serial Config */ @@ -588,7 +577,7 @@ extern "C" { #define meshtastic_ModuleConfig_DetectionSensorConfig_init_default {0, 0, 0, 0, "", 0, _meshtastic_ModuleConfig_DetectionSensorConfig_TriggerType_MIN, 0} #define meshtastic_ModuleConfig_AudioConfig_init_default {0, 0, _meshtastic_ModuleConfig_AudioConfig_Audio_Baud_MIN, 0, 0, 0, 0} #define meshtastic_ModuleConfig_PaxcounterConfig_init_default {0, 0, 0, 0} -#define meshtastic_ModuleConfig_TrafficManagementConfig_init_default {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} +#define meshtastic_ModuleConfig_TrafficManagementConfig_init_default {0, 0, 0, 0, 0} #define meshtastic_ModuleConfig_SerialConfig_init_default {0, 0, 0, 0, _meshtastic_ModuleConfig_SerialConfig_Serial_Baud_MIN, 0, _meshtastic_ModuleConfig_SerialConfig_Serial_Mode_MIN, 0} #define meshtastic_ModuleConfig_ExternalNotificationConfig_init_default {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} #define meshtastic_ModuleConfig_StoreForwardConfig_init_default {0, 0, 0, 0, 0, 0} @@ -607,7 +596,7 @@ extern "C" { #define meshtastic_ModuleConfig_DetectionSensorConfig_init_zero {0, 0, 0, 0, "", 0, _meshtastic_ModuleConfig_DetectionSensorConfig_TriggerType_MIN, 0} #define meshtastic_ModuleConfig_AudioConfig_init_zero {0, 0, _meshtastic_ModuleConfig_AudioConfig_Audio_Baud_MIN, 0, 0, 0, 0} #define meshtastic_ModuleConfig_PaxcounterConfig_init_zero {0, 0, 0, 0} -#define meshtastic_ModuleConfig_TrafficManagementConfig_init_zero {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} +#define meshtastic_ModuleConfig_TrafficManagementConfig_init_zero {0, 0, 0, 0, 0} #define meshtastic_ModuleConfig_SerialConfig_init_zero {0, 0, 0, 0, _meshtastic_ModuleConfig_SerialConfig_Serial_Baud_MIN, 0, _meshtastic_ModuleConfig_SerialConfig_Serial_Mode_MIN, 0} #define meshtastic_ModuleConfig_ExternalNotificationConfig_init_zero {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} #define meshtastic_ModuleConfig_StoreForwardConfig_init_zero {0, 0, 0, 0, 0, 0} @@ -656,20 +645,11 @@ extern "C" { #define meshtastic_ModuleConfig_PaxcounterConfig_paxcounter_update_interval_tag 2 #define meshtastic_ModuleConfig_PaxcounterConfig_wifi_threshold_tag 3 #define meshtastic_ModuleConfig_PaxcounterConfig_ble_threshold_tag 4 -#define meshtastic_ModuleConfig_TrafficManagementConfig_enabled_tag 1 -#define meshtastic_ModuleConfig_TrafficManagementConfig_position_dedup_enabled_tag 2 -#define meshtastic_ModuleConfig_TrafficManagementConfig_position_precision_bits_tag 3 #define meshtastic_ModuleConfig_TrafficManagementConfig_position_min_interval_secs_tag 4 -#define meshtastic_ModuleConfig_TrafficManagementConfig_nodeinfo_direct_response_tag 5 #define meshtastic_ModuleConfig_TrafficManagementConfig_nodeinfo_direct_response_max_hops_tag 6 -#define meshtastic_ModuleConfig_TrafficManagementConfig_rate_limit_enabled_tag 7 #define meshtastic_ModuleConfig_TrafficManagementConfig_rate_limit_window_secs_tag 8 #define meshtastic_ModuleConfig_TrafficManagementConfig_rate_limit_max_packets_tag 9 -#define meshtastic_ModuleConfig_TrafficManagementConfig_drop_unknown_enabled_tag 10 #define meshtastic_ModuleConfig_TrafficManagementConfig_unknown_packet_threshold_tag 11 -#define meshtastic_ModuleConfig_TrafficManagementConfig_exhaust_hop_telemetry_tag 12 -#define meshtastic_ModuleConfig_TrafficManagementConfig_exhaust_hop_position_tag 13 -#define meshtastic_ModuleConfig_TrafficManagementConfig_router_preserve_hops_tag 14 #define meshtastic_ModuleConfig_SerialConfig_enabled_tag 1 #define meshtastic_ModuleConfig_SerialConfig_echo_tag 2 #define meshtastic_ModuleConfig_SerialConfig_rxd_tag 3 @@ -867,20 +847,11 @@ X(a, STATIC, SINGULAR, INT32, ble_threshold, 4) #define meshtastic_ModuleConfig_PaxcounterConfig_DEFAULT NULL #define meshtastic_ModuleConfig_TrafficManagementConfig_FIELDLIST(X, a) \ -X(a, STATIC, SINGULAR, BOOL, enabled, 1) \ -X(a, STATIC, SINGULAR, BOOL, position_dedup_enabled, 2) \ -X(a, STATIC, SINGULAR, UINT32, position_precision_bits, 3) \ X(a, STATIC, SINGULAR, UINT32, position_min_interval_secs, 4) \ -X(a, STATIC, SINGULAR, BOOL, nodeinfo_direct_response, 5) \ X(a, STATIC, SINGULAR, UINT32, nodeinfo_direct_response_max_hops, 6) \ -X(a, STATIC, SINGULAR, BOOL, rate_limit_enabled, 7) \ X(a, STATIC, SINGULAR, UINT32, rate_limit_window_secs, 8) \ X(a, STATIC, SINGULAR, UINT32, rate_limit_max_packets, 9) \ -X(a, STATIC, SINGULAR, BOOL, drop_unknown_enabled, 10) \ -X(a, STATIC, SINGULAR, UINT32, unknown_packet_threshold, 11) \ -X(a, STATIC, SINGULAR, BOOL, exhaust_hop_telemetry, 12) \ -X(a, STATIC, SINGULAR, BOOL, exhaust_hop_position, 13) \ -X(a, STATIC, SINGULAR, BOOL, router_preserve_hops, 14) +X(a, STATIC, SINGULAR, UINT32, unknown_packet_threshold, 11) #define meshtastic_ModuleConfig_TrafficManagementConfig_CALLBACK NULL #define meshtastic_ModuleConfig_TrafficManagementConfig_DEFAULT NULL @@ -1053,7 +1024,7 @@ extern const pb_msgdesc_t meshtastic_RemoteHardwarePin_msg; #define meshtastic_ModuleConfig_StoreForwardConfig_size 24 #define meshtastic_ModuleConfig_TAKConfig_size 4 #define meshtastic_ModuleConfig_TelemetryConfig_size 50 -#define meshtastic_ModuleConfig_TrafficManagementConfig_size 52 +#define meshtastic_ModuleConfig_TrafficManagementConfig_size 30 #define meshtastic_ModuleConfig_size 227 #define meshtastic_RemoteHardwarePin_size 21 diff --git a/src/mesh/mesh-pb-constants.h b/src/mesh/mesh-pb-constants.h index 28dbe3bb7..375be3ae5 100644 --- a/src/mesh/mesh-pb-constants.h +++ b/src/mesh/mesh-pb-constants.h @@ -99,23 +99,20 @@ static inline int get_max_num_nodes() #elif defined(ARCH_PORTDUINO) #define MAX_NUM_NODES 250 // native host: no flash/RAM constraint; match the ESP32-S3 top tier #else -#define MAX_NUM_NODES 120 // nRF52840 (28 KB LittleFS) and generic ESP32 +#define MAX_NUM_NODES 120 // nRF52840 and generic ESP32 (inc. ESP32C3 etc.) #endif // platform #endif // MAX_NUM_NODES /// Per-map cap (position/telemetry/environment/status): only the freshest /// MAX_SATELLITE_NODES nodes keep satellite payloads, the rest just the -/// NodeInfoLite header. RAM-bound: the four maps live in internal SRAM (not -/// PSRAM). PSRAM-equipped ESP32-S3 (and native) keep the full 250; other ESP32 -/// (no-PSRAM, incl. S3) get 80 -- ~32 KB worst case, affordable now the warm -/// tier is trimmed; nRF52840 and other tight parts stay at 40. +/// NodeInfoLite header. RAM-bound (the maps are internal-SRAM, not PSRAM), so +/// flash-rich hosts get a cap >= their hot store (satellites for every node, as +/// before the cap existed) while constrained parts stay at 40. #ifndef MAX_SATELLITE_NODES -#if defined(ARCH_PORTDUINO) || (defined(CONFIG_IDF_TARGET_ESP32S3) && defined(BOARD_HAS_PSRAM)) -#define MAX_SATELLITE_NODES 250 // native / PSRAM-equipped ESP32-S3 -#elif defined(ARCH_ESP32) -#define MAX_SATELLITE_NODES 80 // no-PSRAM ESP32 (incl. ESP32-S3) +#if (defined(CONFIG_IDF_TARGET_ESP32S3) && defined(BOARD_HAS_PSRAM)) || defined(ARCH_PORTDUINO) +#define MAX_SATELLITE_NODES 250 #else -#define MAX_SATELLITE_NODES 40 // nRF52840 (28 KB LittleFS) and other constrained parts +#define MAX_SATELLITE_NODES 40 // nRF52840, generic ESP32, and ESP32-S3 without PSRAM #endif // platform #endif // MAX_SATELLITE_NODES @@ -130,14 +127,12 @@ static inline int get_max_num_nodes() // architecture.h via configuration.h) isn't defined this early in every include // chain. Backed by the raw-flash ring below LittleFS — see WarmNodeStore.h. #define WARM_NODE_COUNT 200 -#elif defined(CONFIG_IDF_TARGET_ESP32S3) && defined(BOARD_HAS_PSRAM) -#define WARM_NODE_COUNT 2000 // ESP32-S3 with PSRAM (external); warm.dat ~80 KB +#elif (defined(CONFIG_IDF_TARGET_ESP32S3) && defined(BOARD_HAS_PSRAM)) || defined(ARCH_PORTDUINO) +#define WARM_NODE_COUNT 2000 // PSRAM-equipped ESP32-S3 / native host; warm.dat ~80 KB #else -// generic ESP32 and no-PSRAM ESP32-S3: ~12.5 KB in internal heap (calloc fallback in -// WarmNodeStore), leaving room for the BLE controller. PSRAM-equipped S3 takes the 2000 case above. -#define WARM_NODE_COUNT 320 -#endif // platform -#endif // WARM_NODE_COUNT +#define WARM_NODE_COUNT 320 // Generic ESP32, ESP32-S3 without PSRAM, ESP32C3 etc. +#endif // platform +#endif // WARM_NODE_COUNT /// Max number of channels allowed #define MAX_NUM_CHANNELS (member_size(meshtastic_ChannelFile, channels) / member_size(meshtastic_ChannelFile, channels[0])) @@ -157,18 +152,21 @@ static inline int get_max_num_nodes() #ifdef ARCH_STM32WL #define HAS_VARIABLE_HOPS 0 #endif + #ifndef HAS_VARIABLE_HOPS #define HAS_VARIABLE_HOPS 1 #endif // Cache size for traffic management (number of nodes to track) -// Can be overridden per-variant based on available memory +// Can be overridden per-variant by defining before this header is included. #ifndef TRAFFIC_MANAGEMENT_CACHE_SIZE -#if HAS_TRAFFIC_MANAGEMENT -#define TRAFFIC_MANAGEMENT_CACHE_SIZE 1000 -#else +#if !HAS_TRAFFIC_MANAGEMENT #define TRAFFIC_MANAGEMENT_CACHE_SIZE 0 -#endif // HAS_TRAFFIC_MANAGEMENT +#elif (defined(CONFIG_IDF_TARGET_ESP32S3) && defined(BOARD_HAS_PSRAM)) || defined(ARCH_PORTDUINO) +#define TRAFFIC_MANAGEMENT_CACHE_SIZE 2048 // PSRAM-equipped ESP32-S3 / native host +#else +#define TRAFFIC_MANAGEMENT_CACHE_SIZE 1000 // Generic ESP32, ESP32-S3 without PSRAM +#endif #endif // TRAFFIC_MANAGEMENT_CACHE_SIZE /// helper function for encoding a record as a protobuf, any failures to encode are fatal and we will panic diff --git a/src/modules/AdminModule.cpp b/src/modules/AdminModule.cpp index af4e3f969..8bcfcd878 100644 --- a/src/modules/AdminModule.cpp +++ b/src/modules/AdminModule.cpp @@ -481,6 +481,8 @@ bool AdminModule::handleReceivedProtobuf(const meshtastic_MeshPacket &mp, meshta screen->setFrames(graphics::Screen::FOCUS_PRESERVE); // <-- Rebuild screens } else if (mp.from == 0) { // local request from the phone — tell the user why it didn't take sendWarning(NodeDB::PROTECTED_CAP_WARN_FMT, "favorite", r->set_favorite_node, MAX_NUM_NODES - 2); + } else { + LOG_WARN("Remote set_favorite_node for 0x%x refused: protected-node cap", r->set_favorite_node); } } break; @@ -508,6 +510,8 @@ bool AdminModule::handleReceivedProtobuf(const meshtastic_MeshPacket &mp, meshta saveChanges(SEGMENT_NODEDATABASE, false); } else if (mp.from == 0) { // local request from the phone — tell the user why it didn't take sendWarning(NodeDB::PROTECTED_CAP_WARN_FMT, "ignore", r->set_ignored_node, MAX_NUM_NODES - 2); + } else { + LOG_WARN("Remote set_ignored_node for 0x%x refused: protected-node cap", r->set_ignored_node); } } break; diff --git a/src/modules/Modules.cpp b/src/modules/Modules.cpp index 5cc94a68f..89c5b9d43 100644 --- a/src/modules/Modules.cpp +++ b/src/modules/Modules.cpp @@ -128,8 +128,7 @@ void setupModules() #endif #if HAS_TRAFFIC_MANAGEMENT && !MESHTASTIC_EXCLUDE_TRAFFIC_MANAGEMENT - // Instantiate only when enabled to avoid extra memory use and background work. - if (moduleConfig.has_traffic_management && moduleConfig.traffic_management.enabled) { + if (moduleConfig.has_traffic_management) { trafficManagementModule = new TrafficManagementModule(); } #endif diff --git a/src/modules/PositionModule.cpp b/src/modules/PositionModule.cpp index ed7215a98..1a0eed933 100644 --- a/src/modules/PositionModule.cpp +++ b/src/modules/PositionModule.cpp @@ -432,6 +432,42 @@ void PositionModule::sendOurPosition(NodeNum dest, bool wantReplies, uint8_t cha #define RUNONCE_INTERVAL 5000; +bool PositionModule::positionUnchangedSinceLastSend(const meshtastic_PositionLite &selfPos, bool useConfiguredPrecision) +{ + if (lastGpsLatitude == 0 && lastGpsLongitude == 0) + return false; // no prior broadcast to compare against + + // Broadcast channel = the one sendOurPosition() would pick (first with non-zero on-wire + // precision). Default nodes gauge movement at that on-wire (public-clamped) resolution; + // trackers use their own configured (unclamped) precision so finer moves still count. + uint32_t precisionBits = 0; + for (uint8_t ch = 0; ch < 8; ch++) { + if (getPositionPrecisionForChannel(ch) == 0) + continue; + precisionBits = + useConfiguredPrecision ? getPositionPrecisionForChannel(channels.getByIndex(ch)) : getPositionPrecisionForChannel(ch); + break; + } + + return positionWithinPrecisionCell(selfPos.latitude_i, selfPos.longitude_i, lastGpsLatitude, lastGpsLongitude, precisionBits); +} + +bool PositionModule::positionWithinPrecisionCell(int32_t aLat, int32_t aLon, int32_t bLat, int32_t bLon, uint32_t precision) +{ + if (precision == 0 || precision >= 32) + return false; // sharing disabled or full precision: no coarse cell to hold within + + return truncateCoordinate(aLat, precision) == truncateCoordinate(bLat, precision) && + truncateCoordinate(aLon, precision) == truncateCoordinate(bLon, precision); +} + +uint32_t PositionModule::effectiveBroadcastIntervalMs(uint32_t configuredIntervalMs, bool stationary, uint32_t stationaryFloorMs) +{ + if (stationary && stationaryFloorMs > configuredIntervalMs) + return stationaryFloorMs; + return configuredIntervalMs; +} + int32_t PositionModule::runOnce() { if (sleepOnNextExecution == true) { @@ -458,7 +494,23 @@ int32_t PositionModule::runOnce() bool waitingForFreshPosition = (lastGpsSend == 0) && !config.position.fixed_position && !nodeDB->hasLocalPositionSinceBoot(); - if (lastGpsSend == 0 || msSinceLastSend >= intervalMs) { + // Hold to the 12h floor when fixed_position (every role: pinning yourself forfeits the + // exception) or when stationary. A real move still goes out early via smart-broadcast below. + // Not-fixed exceptions: lost-and-found broadcasts freely; trackers judge movement at their + // own (unclamped) precision rather than the on-wire one (useConfiguredPrecision). + const auto role = config.device.role; + bool stationary = config.position.fixed_position; + if (!stationary && role != meshtastic_Config_DeviceConfig_Role_LOST_AND_FOUND && nodeDB->hasValidPosition(node)) { + const bool isTracker = + IS_ONE_OF(role, meshtastic_Config_DeviceConfig_Role_TRACKER, meshtastic_Config_DeviceConfig_Role_TAK_TRACKER); + meshtastic_PositionLite selfPos; + if (nodeDB->copyNodePosition(node->num, selfPos)) + stationary = positionUnchangedSinceLastSend(selfPos, /*useConfiguredPrecision=*/isTracker); + } + uint32_t effectiveIntervalMs = + effectiveBroadcastIntervalMs(intervalMs, stationary, (uint32_t)default_position_stationary_broadcast_secs * 1000UL); + + if (lastGpsSend == 0 || msSinceLastSend >= effectiveIntervalMs) { if (waitingForFreshPosition) { #ifdef GPS_DEBUG LOG_DEBUG("Skip initial position send; no fresh position since boot"); diff --git a/src/modules/PositionModule.h b/src/modules/PositionModule.h index d0a4d4603..45535e1b2 100644 --- a/src/modules/PositionModule.h +++ b/src/modules/PositionModule.h @@ -38,6 +38,14 @@ class PositionModule : public ProtobufModule, private concu void handleNewPosition(); + // Pure broadcast-policy helpers, split out so they're unit-testable without the module. + // True when two coordinates truncate to the same precision cell (so a re-broadcast would be a + // duplicate). precision 0 or >=32 returns false: no coarse cell to hold within, never suppress. + static bool positionWithinPrecisionCell(int32_t aLat, int32_t aLon, int32_t bLat, int32_t bLon, uint32_t precision); + // Effective min interval: stationary positions are held to stationaryFloorMs (when that is the + // longer of the two); otherwise the normal configured interval. + static uint32_t effectiveBroadcastIntervalMs(uint32_t configuredIntervalMs, bool stationary, uint32_t stationaryFloorMs); + protected: /** Called to handle a particular incoming message @@ -57,6 +65,12 @@ class PositionModule : public ProtobufModule, private concu private: meshtastic_MeshPacket *allocPositionPacket(); struct SmartPosition getDistanceTraveledSinceLastSend(meshtastic_PositionLite currentPosition); + // True when our position is unchanged since the last broadcast: it truncates to the same + // precision grid cell, so re-sending would be a duplicate that traffic management dedups + // downstream anyway. Used to hold stationary broadcasts to a 12h floor. useConfiguredPrecision + // gauges movement at our own configured (unclamped) precision rather than the on-wire + // (public-clamped) precision — trackers report finer movement. + bool positionUnchangedSinceLastSend(const meshtastic_PositionLite &selfPos, bool useConfiguredPrecision); meshtastic_MeshPacket *allocAtakPli(); void trySetRtc(meshtastic_Position p, bool isLocal, bool forceUpdate = false); uint32_t precision; diff --git a/src/modules/TrafficManagementModule.cpp b/src/modules/TrafficManagementModule.cpp index a89940f2e..95e206e11 100644 --- a/src/modules/TrafficManagementModule.cpp +++ b/src/modules/TrafficManagementModule.cpp @@ -2,9 +2,11 @@ #if HAS_TRAFFIC_MANAGEMENT +#include "Channels.h" #include "Default.h" #include "MeshService.h" #include "NodeDB.h" +#include "PositionPrecision.h" #include "Router.h" #include "TypeConversions.h" #include "airtime.h" @@ -13,6 +15,7 @@ #include "mesh-pb-constants.h" #include "meshUtils.h" #include +#include #include #define TM_LOG_DEBUG(fmt, ...) LOG_DEBUG("[TM] " fmt, ##__VA_ARGS__) @@ -27,7 +30,6 @@ namespace { constexpr uint32_t kMaintenanceIntervalMs = 60 * 1000UL; // Cache cleanup interval -constexpr uint32_t kUnknownResetMs = 60 * 1000UL; // Unknown packet window // NodeInfo direct response: enforced maximum hops by device role // Both use maxHops logic (respond when hopsAway <= threshold) @@ -47,6 +49,17 @@ uint32_t secsToMs(uint32_t secs) return static_cast(ms); } +// Advertised role of the originating node (from NodeDB), or CLIENT (no exception) if unknown. +// Position filtering grants two role exceptions: trackers may refresh duplicates hourly, and +// lost-and-found is throttled only to the shortest dedup window. Both are still subject to +// the channel-precision ceiling in alterReceived(). +meshtastic_Config_DeviceConfig_Role originRole(NodeNum from) +{ + // Resolve via NodeDB: hot store (with user) → warm-tier cached role → CLIENT. The + // warm fallback keeps role exceptions firing for trackers/etc. aged out of the hot store. + return nodeDB ? nodeDB->getNodeRole(from) : meshtastic_Config_DeviceConfig_Role_CLIENT; +} + /** * Clamp precision to a valid dedup range. * Invalid values use the module default precision. @@ -64,57 +77,6 @@ uint8_t sanitizePositionPrecision(uint8_t precision) return 32; } -/** - * Check if a timestamp is within a time window. - * Handles wrap-around correctly using unsigned subtraction. - */ -bool isWithinWindow(uint32_t nowMs, uint32_t startMs, uint32_t intervalMs) -{ - if (intervalMs == 0 || startMs == 0) - return false; - return (nowMs - startMs) < intervalMs; -} - -/** - * Slide an 8-bit relative timestamp back by a wall-clock slab during epoch rebase. - * - * Entries older than the slab clamp to 0 (then reclaimed by the maintenance sweep); - * live entries keep their reconstructed age minus a sub-tick remainder. Each field - * slides by its own resolution's worth of ticks, so a single slab covers all three. - */ -inline void slideRelativeTime(uint8_t &ticks, uint32_t slabMs, uint16_t resolutionSecs) -{ - if (ticks == 0 || resolutionSecs == 0) - return; - uint32_t dec = slabMs / (static_cast(resolutionSecs) * 1000UL); - ticks = (ticks > dec) ? static_cast(ticks - dec) : 0; -} - -/** - * Truncate lat/lon to specified precision for position deduplication. - * - * The truncation works by masking off lower bits and rounding to the center - * of the resulting grid cell. This creates a stable truncated value even - * when GPS jitter causes small coordinate changes. - * - * @param value Raw latitude_i or longitude_i from position - * @param precision Number of significant bits to keep (0-32) - * @return Truncated and centered coordinate value - */ -int32_t truncateLatLon(int32_t value, uint8_t precision) -{ - if (precision == 0 || precision >= 32) - return value; - - // Create mask to zero out lower bits - uint32_t mask = UINT32_MAX << (32 - precision); - uint32_t truncated = static_cast(value) & mask; - - // Add half the truncation step to center in the grid cell - truncated += (1u << (31 - precision)); - return static_cast(truncated); -} - /** * Saturating increment for uint8_t counters. * Prevents overflow by capping at UINT8_MAX (255). @@ -176,23 +138,10 @@ TrafficManagementModule::TrafficManagementModule() : MeshModule("TrafficManageme encryptedOk = true; // Can process encrypted packets stats = meshtastic_TrafficManagementStats_init_zero; - // Initialize rolling epoch for relative timestamps - cacheEpochMs = millis(); - - // Calculate adaptive time resolutions from config (config changes require reboot) - // Resolution = max(60, min(339, interval/2)) for ~24 hour range with good precision - posTimeResolution = calcTimeResolution(Default::getConfiguredOrDefault( - moduleConfig.traffic_management.position_min_interval_secs, default_traffic_mgmt_position_min_interval_secs)); - rateTimeResolution = calcTimeResolution(moduleConfig.traffic_management.rate_limit_window_secs); - unknownTimeResolution = calcTimeResolution(kUnknownResetMs / 1000); // ~5 min default - const auto &cfg = moduleConfig.traffic_management; - TM_LOG_INFO("Enabled: pos_dedup=%d nodeinfo_resp=%d rate_limit=%d drop_unknown=%d exhaust_telem=%d exhaust_pos=%d " - "preserve_hops=%d", - cfg.position_dedup_enabled, cfg.nodeinfo_direct_response, cfg.rate_limit_enabled, cfg.drop_unknown_enabled, - cfg.exhaust_hop_telemetry, cfg.exhaust_hop_position, cfg.router_preserve_hops); - TM_LOG_DEBUG("Time resolutions: pos=%us, rate=%us, unknown=%us", posTimeResolution, rateTimeResolution, - unknownTimeResolution); + TM_LOG_INFO("Config: nodeinfo_max_hops=%u rate_window=%us rate_max=%u unknown_thresh=%u pos_interval=%us", + cfg.nodeinfo_direct_response_max_hops, cfg.rate_limit_window_secs, cfg.rate_limit_max_packets, + cfg.unknown_packet_threshold, cfg.position_min_interval_secs); // Allocate unified cache (10 bytes/entry for all platforms) #if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 @@ -278,9 +227,9 @@ void TrafficManagementModule::resetStats() void TrafficManagementModule::recordRouterHopPreserved() { - if (!moduleConfig.has_traffic_management || !moduleConfig.traffic_management.enabled) - return; - incrementStat(&stats.router_hops_preserved); + // router_preserve_hops: not suitable right now — removed from config until + // the right heuristic for when to preserve vs. exhaust is clearer. + (void)stats.router_hops_preserved; } void TrafficManagementModule::incrementStat(uint32_t *field) @@ -313,6 +262,18 @@ TrafficManagementModule::UnifiedCacheEntry *TrafficManagementModule::findEntry(N #endif } +int TrafficManagementModule::peekCachedRole(NodeNum node) +{ +#if TRAFFIC_MANAGEMENT_CACHE_SIZE == 0 + (void)node; + return -1; +#else + concurrency::LockGuard guard(&cacheLock); + const UnifiedCacheEntry *entry = findEntry(node); + return entry ? static_cast(entry->getCachedRole()) : -1; +#endif +} + /** * Find or create an entry for the given node. * @@ -326,6 +287,51 @@ TrafficManagementModule::UnifiedCacheEntry *TrafficManagementModule::findEntry(N * @param isNew Set to true if a new entry was created * @return Pointer to entry, or nullptr if the cache is unavailable */ +// Sender-role resolution for the position hot path. The tier-3 cache is authoritative +// here and is kept fresh by updateCachedRoleFromNodeInfo() — i.e. updated at the same +// time NodeDB learns a role, not re-derived on every packet. We only fall back to a +// NodeDB scan (tiers 1+2) the first time we start tracking a node, to seed the cache so +// a resident special-role node is correct from its very first position. Thereafter the +// read is O(1) and survives the node aging out of both NodeDB stores. +meshtastic_Config_DeviceConfig_Role TrafficManagementModule::resolveSenderRole(NodeNum from, UnifiedCacheEntry *entry, bool isNew) +{ + if (!entry) + return originRole(from); + if (isNew) { + // First time tracking this node: seed tier 3 from NodeDB (hot → warm). Stores + // CLIENT (0) too, which simply reads back as "no exception". + const meshtastic_Config_DeviceConfig_Role role = originRole(from); + entry->setCachedRole(static_cast(std::min(15, static_cast(role)))); + return role; + } + // Established entry: trust the cached role (refreshed on NodeInfo). No NodeDB scan. + return static_cast(entry->getCachedRole()); +} + +// Refresh the tier-3 role cache from an observed NodeInfo — the same event that updates +// NodeDB's role — so role changes (including demotion back to CLIENT) are picked up +// without scanning NodeDB on the position hot path. Role is read straight from the +// packet's User payload (authoritative regardless of module ordering). Only updates nodes +// we already track (findEntry, no create) so NodeInfo from non-position nodes can't pollute +// the cache; the role rides along with the node's existing position/rate/unknown state. +void TrafficManagementModule::updateCachedRoleFromNodeInfo(const meshtastic_MeshPacket &mp) +{ +#if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 + if (mp.decoded.payload.size == 0) + return; + meshtastic_User user = meshtastic_User_init_zero; + if (!pb_decode_from_bytes(mp.decoded.payload.bytes, mp.decoded.payload.size, &meshtastic_User_msg, &user)) + return; + + concurrency::LockGuard guard(&cacheLock); + UnifiedCacheEntry *entry = findEntry(getFrom(&mp)); + if (entry) + entry->setCachedRole(static_cast(std::min(15, static_cast(user.role)))); +#else + (void)mp; +#endif +} + TrafficManagementModule::UnifiedCacheEntry *TrafficManagementModule::findOrCreateEntry(NodeNum node, bool *isNew) { #if TRAFFIC_MANAGEMENT_CACHE_SIZE == 0 @@ -341,7 +347,7 @@ TrafficManagementModule::UnifiedCacheEntry *TrafficManagementModule::findOrCreat UnifiedCacheEntry *empty = nullptr; UnifiedCacheEntry *victim = nullptr; - bool victimHasHop = true; + bool leastPreferredVictim = true; uint8_t victimRecency = UINT8_MAX; for (uint16_t i = 0; i < cacheSize(); i++) { @@ -355,15 +361,28 @@ TrafficManagementModule::UnifiedCacheEntry *TrafficManagementModule::findOrCreat } if (empty) continue; // an empty slot beats any victim; stop scoring - const bool hasHop = e.next_hop != 0; - uint8_t recency = e.pos_time; - if (e.rate_time > recency) - recency = e.rate_time; - if (e.unknown_time > recency) - recency = e.unknown_time; - if (!victim || (hasHop == victimHasHop ? recency < victimRecency : !hasHop)) { + // "Preferred" entries are evicted last: a confirmed next-hop hint (routing overflow + // store) or a cached special (non-CLIENT) role (tracker / lost-and-found / router). + // Both are the long-tail state this cache exists to retain. + const bool preferred = e.next_hop != 0 || e.getCachedRole() != meshtastic_Config_DeviceConfig_Role_CLIENT; + // Age in pos-ticks (8-bit modular, wraps correctly). Entries with no + // pos state (pos_time==0) score as maximally old (age=currentPosTick()). + const uint8_t nowPosTick = currentPosTick(); + const uint8_t posAge = static_cast(nowPosTick - e.pos_time); + // Blend in rate/unknown ages scaled to pos-tick units (coarser = conservative). + const uint8_t rateAgePosScale = + static_cast(static_cast((currentRateTick() - e.getRateTime()) & 0x0F) * 5 / 3); + const uint8_t unknownAgePosScale = + static_cast(static_cast((currentUnknownTick() - e.getUnknownTime()) & 0x0F) / 6); + uint8_t recencyAge = posAge; + if (e.getRateCount() != 0 && rateAgePosScale > recencyAge) + recencyAge = rateAgePosScale; + if (e.getUnknownCount() != 0 && unknownAgePosScale > recencyAge) + recencyAge = unknownAgePosScale; + const uint8_t recency = static_cast(UINT8_MAX - recencyAge); + if (!victim || (preferred == leastPreferredVictim ? recency < victimRecency : !preferred)) { victim = &e; - victimHasHop = hasHop; + leastPreferredVictim = preferred; victimRecency = recency; } } @@ -417,7 +436,7 @@ TrafficManagementModule::NodeInfoPayloadEntry *TrafficManagementModule::findOrCr NodeInfoPayloadEntry *empty = nullptr; NodeInfoPayloadEntry *lru = nullptr; uint32_t lruAge = 0; - const uint32_t now = millis(); + const uint32_t now = clockMs(); for (uint16_t i = 0; i < nodeInfoTargetEntries(); i++) { NodeInfoPayloadEntry &e = nodeInfoPayload[i]; @@ -493,7 +512,7 @@ void TrafficManagementModule::cacheNodeInfoPacket(const meshtastic_MeshPacket &m // richer context than "just the user protobuf" when PSRAM is present. // This path is intentionally independent from NodeInfoModule/NodeDB. entry->user = user; - entry->lastObservedMs = millis(); + entry->lastObservedMs = clockMs(); entry->lastObservedRxTime = mp.rx_time; entry->sourceChannel = mp.channel; entry->hasDecodedBitfield = mp.decoded.has_bitfield; @@ -570,11 +589,11 @@ void TrafficManagementModule::clearNextHop(NodeNum dest) #endif } -void TrafficManagementModule::preloadNextHopsFromNodeDB() +bool TrafficManagementModule::preloadNextHopsFromNodeDB() { #if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 if (!cache || !nodeDB) - return; + return false; // prerequisites not ready yet — caller should retry on a later pass uint16_t seeded = 0; concurrency::LockGuard guard(&cacheLock); @@ -594,6 +613,9 @@ void TrafficManagementModule::preloadNextHopsFromNodeDB() } TM_LOG_INFO("Preloaded %u next-hop hints from NodeDB", static_cast(seeded)); + return true; +#else + return true; // nothing to preload on a cache-less build; don't keep retrying #endif } @@ -601,59 +623,11 @@ void TrafficManagementModule::preloadNextHopsFromNodeDB() // Epoch Management // ============================================================================= -/** - * Reset the timestamp epoch when relative offsets approach overflow. - * - * Called when epoch age exceeds ~19 hours (approaching 8-bit minute overflow). - * Invalidates all cached per-node traffic state. - */ -void TrafficManagementModule::resetEpoch(uint32_t nowMs) +void TrafficManagementModule::flushCache() { #if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 - TM_LOG_DEBUG("Resetting cache epoch"); - cacheEpochMs = nowMs; - - // Full flush avoids stale dedup identity/counters surviving epoch rollover. + TM_LOG_DEBUG("Flushing cache"); memset(cache, 0, static_cast(cacheSize()) * sizeof(UnifiedCacheEntry)); -#else - (void)nowMs; -#endif -} - -/** - * Sliding-epoch rebase — preserve cached state past the 8-bit timestamp horizon. - * - * Instead of flushing the whole cache when offsets approach overflow, advance the - * epoch by a fixed slab and shift every live entry's relative timestamps back by - * the same wall-clock amount. A valid entry's window is only a handful of ticks - * wide (TTL auto-scales with resolution), so live entries comfortably survive; - * already-expired entries clamp to 0 and are reclaimed by the maintenance sweep in - * the same locked pass. Reconstructed absolute time is preserved (minus a sub-tick - * remainder), so in-flight TTL checks remain correct across the rebase. - * - * Caller must hold cacheLock. - */ -void TrafficManagementModule::rebaseEpoch(uint32_t nowMs) -{ -#if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 - (void)nowMs; - - // Slab stays well below the 200-tick reset threshold so a single rebase drops - // the offset back into range (~200 -> ~72 ticks) while live entries survive. - const uint32_t slabMs = 128UL * maxResolution() * 1000UL; - cacheEpochMs += slabMs; - - TM_LOG_DEBUG("Rebasing cache epoch by %lus", static_cast(slabMs / 1000UL)); - - for (uint16_t i = 0; i < cacheSize(); i++) { - if (cache[i].node == 0) - continue; - slideRelativeTime(cache[i].pos_time, slabMs, posTimeResolution); - slideRelativeTime(cache[i].rate_time, slabMs, rateTimeResolution); - slideRelativeTime(cache[i].unknown_time, slabMs, unknownTimeResolution); - } -#else - (void)nowMs; #endif } @@ -697,7 +671,12 @@ uint8_t TrafficManagementModule::computePositionFingerprint(int32_t lat_truncate uint8_t latBits = (static_cast(lat_truncated) >> shift) & ((1u << bitsToTake) - 1); uint8_t lonBits = (static_cast(lon_truncated) >> shift) & ((1u << bitsToTake) - 1); - return static_cast((latBits << 4) | lonBits); + const uint8_t fp = static_cast((latBits << 4) | lonBits); + // 0 is the "no position seen" sentinel for pos_fingerprint, so a real position that happens to + // hash to 0 must not collide with it (otherwise its duplicates would never dedup). Remap 0 -> 0xFF, + // mirroring NodeDB::getLastByteOfNodeNum()'s 0 -> 0xFF idiom. Cost: the 0x00 bucket merges into + // 0xFF (one extra collision in 256 — negligible; the fingerprint already collides every 16 cells). + return fp ? fp : 0xFF; } // ============================================================================= @@ -712,7 +691,7 @@ uint8_t TrafficManagementModule::computePositionFingerprint(int32_t lat_truncate // force hop_limit=0 on the rebroadcast copy, allowing one final relay hop. ProcessMessage TrafficManagementModule::handleReceived(const meshtastic_MeshPacket &mp) { - if (!moduleConfig.has_traffic_management || !moduleConfig.traffic_management.enabled) + if (!moduleConfig.has_traffic_management) return ProcessMessage::CONTINUE; ignoreRequest = false; @@ -722,7 +701,7 @@ ProcessMessage TrafficManagementModule::handleReceived(const meshtastic_MeshPack incrementStat(&stats.packets_inspected); const auto &cfg = moduleConfig.traffic_management; - const uint32_t nowMs = millis(); + const uint32_t nowMs = TrafficManagementModule::clockMs(); // ------------------------------------------------------------------------- // Undecoded Packet Handling @@ -731,7 +710,7 @@ ProcessMessage TrafficManagementModule::handleReceived(const meshtastic_MeshPack // a misbehaving node. Track and optionally drop repeat offenders. if (mp.which_payload_variant != meshtastic_MeshPacket_decoded_tag) { - if (cfg.drop_unknown_enabled && cfg.unknown_packet_threshold > 0) { + if (cfg.unknown_packet_threshold > 0) { if (shouldDropUnknown(&mp, nowMs)) { logAction("drop", &mp, "unknown"); incrementStat(&stats.unknown_packet_drops); @@ -742,9 +721,12 @@ ProcessMessage TrafficManagementModule::handleReceived(const meshtastic_MeshPack return ProcessMessage::CONTINUE; } - // Learn NodeInfo payloads into the dedicated PSRAM cache. - if (mp.decoded.portnum == meshtastic_PortNum_NODEINFO_APP) + // Learn NodeInfo payloads into the dedicated PSRAM cache, and refresh the tier-3 + // role cache for any node we already track (keeps the dedup role exception current). + if (mp.decoded.portnum == meshtastic_PortNum_NODEINFO_APP) { cacheNodeInfoPacket(mp); + updateCachedRoleFromNodeInfo(mp); + } // ------------------------------------------------------------------------- // NodeInfo Direct Response @@ -754,8 +736,8 @@ ProcessMessage TrafficManagementModule::handleReceived(const meshtastic_MeshPack // STOP prevents the request from being rebroadcast toward the target node, // and our cached response is sent back to the requestor with hop_limit=0. - if (cfg.nodeinfo_direct_response && mp.decoded.portnum == meshtastic_PortNum_NODEINFO_APP && mp.decoded.want_response && - !isBroadcast(mp.to) && !isToUs(&mp) && !isFromUs(&mp)) { + if (cfg.nodeinfo_direct_response_max_hops > 0 && mp.decoded.portnum == meshtastic_PortNum_NODEINFO_APP && + mp.decoded.want_response && !isBroadcast(mp.to) && !isToUs(&mp) && !isFromUs(&mp)) { if (shouldRespondToNodeInfo(&mp, true)) { meshtastic_User requester = meshtastic_User_init_zero; if (pb_decode_from_bytes(mp.decoded.payload.bytes, mp.decoded.payload.size, &meshtastic_User_msg, &requester)) { @@ -776,7 +758,7 @@ ProcessMessage TrafficManagementModule::handleReceived(const meshtastic_MeshPack // GPS jitter within the configured precision. if (!isFromUs(&mp) && !isToUs(&mp)) { - if (cfg.position_dedup_enabled && mp.decoded.portnum == meshtastic_PortNum_POSITION_APP) { + if (channels.isWellKnownChannel(mp.channel) && mp.decoded.portnum == meshtastic_PortNum_POSITION_APP) { meshtastic_Position pos = meshtastic_Position_init_zero; if (pb_decode_from_bytes(mp.decoded.payload.bytes, mp.decoded.payload.size, &meshtastic_Position_msg, &pos)) { if (shouldDropPosition(&mp, &pos, nowMs)) { @@ -794,7 +776,7 @@ ProcessMessage TrafficManagementModule::handleReceived(const meshtastic_MeshPack // Throttle nodes sending too many packets within a time window. // Excludes routing and admin packets which are essential for mesh operation. - if (cfg.rate_limit_enabled && cfg.rate_limit_window_secs > 0 && cfg.rate_limit_max_packets > 0) { + if (cfg.rate_limit_window_secs > 0 && cfg.rate_limit_max_packets > 0) { if (mp.decoded.portnum != meshtastic_PortNum_ROUTING_APP && mp.decoded.portnum != meshtastic_PortNum_ADMIN_APP) { if (isRateLimited(mp.from, nowMs)) { logAction("drop", &mp, "rate-limit"); @@ -811,7 +793,7 @@ ProcessMessage TrafficManagementModule::handleReceived(const meshtastic_MeshPack void TrafficManagementModule::alterReceived(meshtastic_MeshPacket &mp) { - if (!moduleConfig.has_traffic_management || !moduleConfig.traffic_management.enabled) + if (!moduleConfig.has_traffic_management) return; if (mp.which_payload_variant != meshtastic_MeshPacket_decoded_tag) @@ -820,40 +802,45 @@ void TrafficManagementModule::alterReceived(meshtastic_MeshPacket &mp) if (isFromUs(&mp)) return; - // ------------------------------------------------------------------------- - // Relayed Broadcast Hop Exhaustion - // ------------------------------------------------------------------------- - // For relayed telemetry or position broadcasts from other nodes, optionally - // set hop_limit=0 so they don't propagate further through the mesh. + // exhaust_hop_telemetry / exhaust_hop_position / router_preserve_hops: + // not suitable right now — the right heuristics for when to exhaust or + // preserve hops need more field data before we expose them as config knobs. + // exhaustRequested stays false; perhapsRebroadcast() behaves normally. - const auto &cfg = moduleConfig.traffic_management; - const bool isTelemetry = mp.decoded.portnum == meshtastic_PortNum_TELEMETRY_APP; const bool isPosition = mp.decoded.portnum == meshtastic_PortNum_POSITION_APP; - // Only exhaust telemetry hops when channel is actually congested, mirroring the same - // airtime checks that gate self-generated telemetry in the telemetry modules. - const bool channelBusy = airTime && (!airTime->isTxAllowedChannelUtil(true) || !airTime->isTxAllowedAirUtil()); - const bool shouldExhaust = - ((channelBusy && isTelemetry && cfg.exhaust_hop_telemetry) || (isPosition && cfg.exhaust_hop_position)); - if (!shouldExhaust || !isBroadcast(mp.to)) - return; - - if (mp.hop_limit > 0) { - const char *reason = isTelemetry ? "exhaust-hop-telemetry" : "exhaust-hop-position"; - logAction("exhaust", &mp, reason); - // Adjust hop_start so downstream nodes compute correct hopsAway (hop_start - hop_limit). - // Without this, hop_limit=0 with original hop_start would show inflated hopsAway. - mp.hop_start = mp.hop_start - mp.hop_limit + 1; - mp.hop_limit = 0; - // Signal perhapsRebroadcast() to allow one final relay with hop_limit=0. - // Without this flag, perhapsRebroadcast() would skip the packet since hop_limit==0. - // The packet-scoped flag is checked in NextHopRouter::perhapsRebroadcast() - // and forces tosend->hop_limit=0, ensuring no further propagation beyond the - // next node. - exhaustRequested = true; - exhaustRequestedFrom = getFrom(&mp); - exhaustRequestedId = mp.id; - incrementStat(&stats.hop_exhausted_packets); + // ------------------------------------------------------------------------- + // Relayed Position Precision Clamp + // ------------------------------------------------------------------------- + // Clamp relayed position broadcasts to the channel's configured precision + // ceiling. Guards against forwarding more-precise coordinates than the + // channel is intended to carry (e.g. a LongFast channel set to 13-bit / + // ~1.5 km). chanPrec==0 means position sharing is disabled on the channel; + // skip — not our job to zero positions on relay. + // Ham mode (owner.is_licensed) is exempt. Lost-and-found is NOT exempt — its relayed + // positions get the same precision clamp as any node. + // Compile USERPREFS_TMM_APPLY_TO_PRIVATE_CHANNELS to extend to private channels. + if (!owner.is_licensed && isPosition && isBroadcast(mp.to)) { +#ifdef USERPREFS_TMM_APPLY_TO_PRIVATE_CHANNELS + const bool shouldClamp = true; +#else + const bool shouldClamp = channels.isWellKnownChannel(mp.channel); +#endif + if (shouldClamp) { + const uint32_t chanPrec = getPositionPrecisionForChannel(mp.channel); + if (chanPrec > 0) { + meshtastic_Position pos = meshtastic_Position_init_default; + if (pb_decode_from_bytes(mp.decoded.payload.bytes, mp.decoded.payload.size, &meshtastic_Position_msg, &pos)) { + const uint32_t packetPrec = pos.precision_bits > 0 ? pos.precision_bits : 32u; + if (packetPrec > chanPrec) { + applyPositionPrecision(pos, chanPrec); + mp.decoded.payload.size = pb_encode_to_bytes(mp.decoded.payload.bytes, sizeof(mp.decoded.payload.bytes), + &meshtastic_Position_msg, &pos); + logAction("clamp", &mp, "precision"); + } + } + } + } } } @@ -863,53 +850,58 @@ void TrafficManagementModule::alterReceived(meshtastic_MeshPacket &mp) int32_t TrafficManagementModule::runOnce() { - if (!moduleConfig.has_traffic_management || !moduleConfig.traffic_management.enabled) + if (!moduleConfig.has_traffic_management) return INT32_MAX; #if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 - const uint32_t nowMs = millis(); + const uint32_t nowMs = TrafficManagementModule::clockMs(); // Warm-start the next-hop cache from persisted NodeInfoLite hints once nodeDB // is populated. Done here (not in the constructor) so nodeDB has finished // loading. Takes its own lock, so call before acquiring the sweep guard below. - if (!nextHopPreloaded) { - preloadNextHopsFromNodeDB(); + // Only latch the one-shot guard once the preload actually ran; if nodeDB wasn't + // ready yet, retry on the next maintenance pass instead of skipping it forever. + if (!nextHopPreloaded && preloadNextHopsFromNodeDB()) nextHopPreloaded = true; - } - // Calculate TTLs for cache expiration + // Free-running tick counters (no epoch needed). + // TTL expressed in ticks: + // pos: 4× position_min_interval_secs (clamped to 255 ticks @ 6 min/tick) + // rate: 2× rate_limit_window_secs (clamped to 15 ticks @ 5 min/tick; only relevant when rate limits are configured) + // unknown: fixed 12 ticks @ 1 min/tick (only relevant when unknown_packet_threshold > 0) const uint32_t positionIntervalMs = secsToMs(Default::getConfiguredOrDefault( moduleConfig.traffic_management.position_min_interval_secs, default_traffic_mgmt_position_min_interval_secs)); - const uint32_t positionTtlMs = positionIntervalMs * 4; + const uint8_t posTtlTicks = + static_cast(std::min(static_cast(255), (positionIntervalMs * 4) / kPosTimeTickMs)); - const uint32_t rateIntervalMs = secsToMs(moduleConfig.traffic_management.rate_limit_window_secs); - const uint32_t rateTtlMs = (rateIntervalMs > 0) ? rateIntervalMs * 2 : (10 * 60 * 1000UL); + const uint32_t rateWindowMs = secsToMs(moduleConfig.traffic_management.rate_limit_window_secs); + const uint8_t rateTtlTicks = static_cast( + std::min(static_cast(15), (rateWindowMs > 0 ? rateWindowMs * 2 : 24 * kRateTimeTickMs) / kRateTimeTickMs)); - const uint32_t unknownTtlMs = kUnknownResetMs * 5; + // unknown: fixed 12-tick TTL (12 min — 4 ticks past the 5-min default window) + const uint8_t unknownTtlTicks = 12; + + const uint8_t nowPosTick = currentPosTick(); + const uint8_t nowRateTick = currentRateTick(); + const uint8_t nowUnknownTick = currentUnknownTick(); // Sweep cache and clear expired entries uint16_t activeEntries = 0; uint16_t expiredEntries = 0; - const uint32_t sweepStartMs = millis(); + const uint32_t sweepStartMs = TrafficManagementModule::clockMs(); + const auto &cfg = moduleConfig.traffic_management; concurrency::LockGuard guard(&cacheLock); - // Slide the epoch instead of flushing when offsets approach 8-bit overflow. - // Rebase preserves live entries; only already-expired ones clamp to 0 and are - // reclaimed by the sweep below in this same locked pass. - if (needsEpochReset(nowMs)) - rebaseEpoch(nowMs); - for (uint16_t i = 0; i < cacheSize(); i++) { if (cache[i].node == 0) continue; bool anyValid = false; - // Check and clear expired position data - if (cache[i].pos_time != 0) { - uint32_t posTimeMs = fromRelativePosTime(cache[i].pos_time); - if (!isWithinWindow(nowMs, posTimeMs, positionTtlMs)) { + // Check and clear expired position data (presence: pos_fingerprint != 0) + if (cache[i].pos_fingerprint != 0) { + if (static_cast(nowPosTick - cache[i].pos_time) >= posTtlTicks) { cache[i].pos_fingerprint = 0; cache[i].pos_time = 0; } else { @@ -917,31 +909,33 @@ int32_t TrafficManagementModule::runOnce() } } - // Check and clear expired rate limit data - if (cache[i].rate_time != 0) { - uint32_t rateTimeMs = fromRelativeRateTime(cache[i].rate_time); - if (!isWithinWindow(nowMs, rateTimeMs, rateTtlMs)) { - cache[i].rate_count = 0; - cache[i].rate_time = 0; + // Check and clear expired rate limit data (presence: getRateCount() != 0) + if (cache[i].getRateCount() != 0) { + if ((static_cast(nowRateTick - cache[i].getRateTime()) & 0x0F) >= rateTtlTicks) { + cache[i].setRateCount(0); + cache[i].setRateTime(0); } else { anyValid = true; } } - // Check and clear expired unknown tracking data - if (cache[i].unknown_time != 0) { - uint32_t unknownTimeMs = fromRelativeUnknownTime(cache[i].unknown_time); - if (!isWithinWindow(nowMs, unknownTimeMs, unknownTtlMs)) { - cache[i].unknown_count = 0; - cache[i].unknown_time = 0; + // Check and clear expired unknown tracking data (presence: getUnknownCount() != 0) + if (cache[i].getUnknownCount() != 0) { + if ((static_cast(nowUnknownTick - cache[i].getUnknownTime()) & 0x0F) >= unknownTtlTicks) { + cache[i].setUnknownCount(0); + cache[i].setUnknownTime(0); } else { anyValid = true; } } - // A confirmed next-hop hint has no TTL of its own and keeps the slot alive, - // so an aged-out routing hint outlives the dedup/rate/unknown state. - if (cache[i].next_hop != 0) + // Two fields have no TTL of their own and pin the slot, so they outlive the + // dedup/rate/unknown state: + // - a confirmed next-hop hint (the routing overflow store), and + // - a cached special (non-CLIENT) role, so a tracker / lost-and-found / router + // keeps its dedup-window exception across quiet periods rather than reverting + // to CLIENT the moment its timed state expires. + if (cache[i].next_hop != 0 || cache[i].getCachedRole() != meshtastic_Config_DeviceConfig_Role_CLIENT) anyValid = true; // If all data expired, free the slot entirely @@ -955,7 +949,7 @@ int32_t TrafficManagementModule::runOnce() TM_LOG_DEBUG("Maintenance: %u active, %u expired, %u/%u slots, %lums elapsed", activeEntries, expiredEntries, static_cast(activeEntries), static_cast(cacheSize()), - static_cast(millis() - sweepStartMs)); + static_cast(TrafficManagementModule::clockMs() - sweepStartMs)); #if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) if (nodeInfoPayload) { @@ -984,18 +978,21 @@ bool TrafficManagementModule::shouldDropPosition(const meshtastic_MeshPacket *p, if (!pos->has_latitude_i || !pos->has_longitude_i) return false; - uint8_t precision = Default::getConfiguredOrDefault(moduleConfig.traffic_management.position_precision_bits, - default_traffic_mgmt_position_precision_bits); - precision = sanitizePositionPrecision(precision); + // Precision is driven by the channel's own position_precision ceiling — the same + // grid the channel uses for broadcast. Falls back to the firmware default (19-bit, + // ~90m cells) when the channel has no precision configured (chanPrec == 0). + const uint32_t chanPrec = getPositionPrecisionForChannel(p->channel); + uint8_t precision = sanitizePositionPrecision( + chanPrec > 0 ? static_cast(chanPrec) : static_cast(default_traffic_mgmt_position_precision_bits)); - const int32_t lat_truncated = truncateLatLon(pos->latitude_i, precision); - const int32_t lon_truncated = truncateLatLon(pos->longitude_i, precision); + const int32_t lat_truncated = truncateCoordinate(pos->latitude_i, precision); + const int32_t lon_truncated = truncateCoordinate(pos->longitude_i, precision); const uint8_t fingerprint = computePositionFingerprint(lat_truncated, lon_truncated, precision); // Drop gate uses the RAW configured interval: 0 means "dedup disabled" (the // contract documented below). The 12h default is only for resolution/TTL // sizing (constructor / runOnce), not for deciding whether to drop — feeding // the default here would silently turn the 0-disables-dedup contract off. - const uint32_t minIntervalMs = secsToMs(moduleConfig.traffic_management.position_min_interval_secs); + uint32_t minIntervalMs = secsToMs(moduleConfig.traffic_management.position_min_interval_secs); bool isNew = false; concurrency::LockGuard guard(&cacheLock); @@ -1003,19 +1000,46 @@ bool TrafficManagementModule::shouldDropPosition(const meshtastic_MeshPacket *p, if (!entry) return false; - // Compare fingerprint and check time window - // When minIntervalMs == 0, deduplication is disabled (withinInterval = false means never drop) - const bool hasPositionState = !isNew && entry->pos_time != 0; + // Role exceptions keyed on the originating node's advertised role, resolved across + // all three tiers (hot store → warm store → TMM live cache). The position path is + // the one place that needs sender-role, and it also keeps tier 3 warm so the + // exception survives the node aging out of both NodeDB stores — important in the + // common dedup-only config, where isRateLimited()'s role write never runs. + const meshtastic_Config_DeviceConfig_Role role = resolveSenderRole(p->from, entry, isNew); + if (role == meshtastic_Config_DeviceConfig_Role_LOST_AND_FOUND) { + // Lost-and-found may refresh a duplicate position at most every ~15 min (cap, never + // lengthens; quantised to ~2 dedup ticks). Only when dedup is active — never tighten + // past an operator who disabled it (0). + const uint32_t lostFoundCapMs = secsToMs(default_traffic_mgmt_lost_and_found_position_min_interval_secs); + if (minIntervalMs != 0 && minIntervalMs > lostFoundCapMs) + minIntervalMs = lostFoundCapMs; + } else if (role == meshtastic_Config_DeviceConfig_Role_TRACKER || role == meshtastic_Config_DeviceConfig_Role_TAK_TRACKER) { + // Trackers may refresh a duplicate position as often as hourly (cap, never lengthens). + const uint32_t trackerCapMs = secsToMs(default_traffic_mgmt_tracker_position_min_interval_secs); + if (minIntervalMs > trackerCapMs) + minIntervalMs = trackerCapMs; + } + + // Compare fingerprint and check time window. + // When minIntervalMs == 0, deduplication is disabled (withinInterval = false means never drop). + // Presence: pos_fingerprint != 0; computePositionFingerprint() remaps 0 -> 0xFF so zero means unseen. + const bool hasPositionState = !isNew && entry->pos_fingerprint != 0; const bool samePosition = hasPositionState && entry->pos_fingerprint == fingerprint; + const uint8_t nowPosTick = currentPosTick(); + // Clamp to [1, 255]: intervals shorter than one tick still dedup within the same tick. + const uint8_t windowTicks = + (minIntervalMs == 0) ? 0 + : static_cast(std::min(static_cast(UINT8_MAX), + std::max(static_cast(1), minIntervalMs / kPosTimeTickMs))); const bool withinInterval = - hasPositionState && (minIntervalMs != 0) && isWithinWindow(nowMs, fromRelativePosTime(entry->pos_time), minIntervalMs); + hasPositionState && (windowTicks != 0) && (static_cast(nowPosTick - entry->pos_time) < windowTicks); TM_LOG_DEBUG("Position dedup 0x%08x: fp=0x%02x prev=0x%02x same=%d within=%d new=%d", p->from, fingerprint, entry->pos_fingerprint, samePosition, withinInterval, isNew); - // Update cache entry + // Update cache entry (raw tick; 0 is a valid tick value) entry->pos_fingerprint = fingerprint; - entry->pos_time = toRelativePosTime(nowMs); + entry->pos_time = nowPosTick; // Drop only if same position AND within the minimum interval return samePosition && withinInterval; @@ -1104,7 +1128,7 @@ bool TrafficManagementModule::shouldRespondToNodeInfo(const meshtastic_MeshPacke reply->decoded.bitfield |= BITFIELD_OK_TO_MQTT_MASK; if (hasCachedUser && cachedLastObservedMs != 0) { - uint32_t ageMs = millis() - cachedLastObservedMs; + uint32_t ageMs = clockMs() - cachedLastObservedMs; TM_LOG_DEBUG("NodeInfo PSRAM hit node=0x%08x age=%lu ms src_ch=%u req_ch=%u rx_time=%lu", p->to, static_cast(ageMs), static_cast(cachedSourceChannel), static_cast(p->channel), static_cast(cachedLastObservedRxTime)); @@ -1169,25 +1193,32 @@ bool TrafficManagementModule::isRateLimited(NodeNum from, uint32_t nowMs) if (!entry) return false; - // Check if window has expired - if (isNew || !isWithinWindow(nowMs, fromRelativeRateTime(entry->rate_time), windowMs)) { - entry->rate_time = toRelativeRateTime(nowMs); - entry->rate_count = 1; + // Window ticks: clamp to [1,15] so zero windowMs (config error) opens a new window. + const uint8_t windowTicks = static_cast(std::min(static_cast(15), windowMs / kRateTimeTickMs)); + const uint8_t nowRateTick = currentRateTick(); + const bool windowExpired = + isNew || entry->getRateCount() == 0 || + ((static_cast(nowRateTick - entry->getRateTime()) & 0x0F) >= std::max(static_cast(1), windowTicks)); + if (windowExpired) { + entry->setRateTime(nowRateTick); + entry->setRateCount(1); return false; } - // Increment counter (saturates at 255) - saturatingIncrement(entry->rate_count); + // Increment counter, saturating at 63 (6-bit field max). + const uint8_t cur = entry->getRateCount(); + if (cur < 0x3F) + entry->setRateCount(static_cast(cur + 1)); - // Check against threshold (uint8_t max is 255, but config is uint32_t) + // Threshold capped at 60 so a saturated reading (63) always exceeds it. uint32_t threshold = moduleConfig.traffic_management.rate_limit_max_packets; - if (threshold > 255) - threshold = 255; + if (threshold > 60) + threshold = 60; - bool limited = entry->rate_count > threshold; - if (limited || entry->rate_count == threshold) { - TM_LOG_DEBUG("Rate limit 0x%08x: count=%u threshold=%u -> %s", from, entry->rate_count, threshold, - limited ? "DROP" : "at-limit"); + const uint8_t count = entry->getRateCount(); + bool limited = count > threshold; + if (limited || count == threshold) { + TM_LOG_DEBUG("Rate limit 0x%08x: count=%u threshold=%u -> %s", from, count, threshold, limited ? "DROP" : "at-limit"); } return limited; #endif @@ -1200,12 +1231,11 @@ bool TrafficManagementModule::shouldDropUnknown(const meshtastic_MeshPacket *p, (void)nowMs; return false; #else - if (!moduleConfig.traffic_management.drop_unknown_enabled || moduleConfig.traffic_management.unknown_packet_threshold == 0) + if (moduleConfig.traffic_management.unknown_packet_threshold == 0) return false; - uint32_t windowMs = kUnknownResetMs; - if (moduleConfig.traffic_management.rate_limit_window_secs > 0) - windowMs = secsToMs(moduleConfig.traffic_management.rate_limit_window_secs); + // Fixed 5-tick (5 min) unknown window; capped at 12 ticks (12 min max). + static constexpr uint8_t kUnknownWindowTicks = 5; bool isNew = false; concurrency::LockGuard guard(&cacheLock); @@ -1213,25 +1243,31 @@ bool TrafficManagementModule::shouldDropUnknown(const meshtastic_MeshPacket *p, if (!entry) return false; - // Check if window has expired - if (isNew || !isWithinWindow(nowMs, fromRelativeUnknownTime(entry->unknown_time), windowMs)) { - entry->unknown_time = toRelativeUnknownTime(nowMs); - entry->unknown_count = 0; + // Check if window has expired (presence: getUnknownCount() != 0) + const uint8_t nowUnknownTick = currentUnknownTick(); + const bool windowExpired = isNew || entry->getUnknownCount() == 0 || + ((static_cast(nowUnknownTick - entry->getUnknownTime()) & 0x0F) >= kUnknownWindowTicks); + if (windowExpired) { + entry->setUnknownTime(nowUnknownTick); + entry->setUnknownCount(0); } - // Increment counter (saturates at 255). Same saturation handling as - // isRateLimited: without it, a clamped threshold of 255 can never fire. - const bool alreadySaturated = (entry->unknown_count == UINT8_MAX); - saturatingIncrement(entry->unknown_count); + // Increment counter, saturating at 63 (6-bit field max). With threshold + // capped at 60, a saturated reading always exceeds the limit — no special + // already-saturated edge case needed. + const uint8_t cur = entry->getUnknownCount(); + if (cur < 0x3F) + entry->setUnknownCount(static_cast(cur + 1)); - // Check against threshold + // Threshold capped at 60 so a saturated reading (63) always exceeds it. uint32_t threshold = moduleConfig.traffic_management.unknown_packet_threshold; - if (threshold > 255) - threshold = 255; + if (threshold > 60) + threshold = 60; - bool drop = entry->unknown_count > threshold || (alreadySaturated && threshold == 255); - if (drop || entry->unknown_count == threshold) { - TM_LOG_DEBUG("Unknown packets 0x%08x: count=%u threshold=%u -> %s", p->from, entry->unknown_count, threshold, + const uint8_t count = entry->getUnknownCount(); + bool drop = count > threshold; + if (drop || count == threshold) { + TM_LOG_DEBUG("Unknown packets 0x%08x: count=%u threshold=%u -> %s", p->from, count, threshold, drop ? "DROP" : "at-limit"); } return drop; diff --git a/src/modules/TrafficManagementModule.h b/src/modules/TrafficManagementModule.h index b0e97d89c..231849e8f 100644 --- a/src/modules/TrafficManagementModule.h +++ b/src/modules/TrafficManagementModule.h @@ -22,9 +22,10 @@ * Memory Optimization: * Uses one flat unified cache (plain array, linear scan) shared by all * per-node features instead of separate per-feature caches. Timestamps are - * stored as 8-bit relative offsets from a rolling epoch to further reduce - * memory footprint. LoRa packet rates are low enough that an O(n) scan of - * ~1000 11-byte entries is negligible next to packet processing. + * stored as free-running modular tick counters (pos: 8-bit 360 s/tick; + * rate+unknown: paired 4-bit nibbles in one byte) for a 10-byte entry. + * LoRa packet rates are low enough that an O(n) scan of ~1000 entries is + * negligible next to packet processing. */ class TrafficManagementModule : public MeshModule, private concurrency::OSThread { @@ -54,7 +55,9 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread // Warm-start the next-hop cache from persisted NodeInfoLite hints so confirmed // hops survive later hot-store (NodeDB) eviction. Idempotent; runs once after // nodeDB is populated (lazily on first maintenance pass). - void preloadNextHopsFromNodeDB(); + // @return true if it actually ran (prereqs met / nothing to do); false if + // prerequisites (cache, nodeDB) weren't ready yet, so the caller should retry. + bool preloadNextHopsFromNodeDB(); /** * Check if this packet should have its hops exhausted. @@ -66,81 +69,90 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread return exhaustRequested && exhaustRequestedFrom == getFrom(&mp) && exhaustRequestedId == mp.id; } + // Injectable monotonic clock (ms). All TMM time reads go through clockMs() so unit tests can + // advance a virtual timebase instead of sleeping real seconds across the 6 min/360 s tick. + // Mirrors HopScalingModule::s_testNowMs. Writable from tests as TrafficManagementModule::s_testNowMs; + // ignored in production (clockMs() returns millis()). + inline static uint32_t s_testNowMs = 0; +#ifdef PIO_UNIT_TESTING + static uint32_t clockMs() { return s_testNowMs; } +#else + static uint32_t clockMs() { return millis(); } +#endif + protected: ProcessMessage handleReceived(const meshtastic_MeshPacket &mp) override; bool wantPacket(const meshtastic_MeshPacket *p) override { return true; } void alterReceived(meshtastic_MeshPacket &mp) override; int32_t runOnce() override; - // Protected so test shims can force epoch rollover behavior. - void resetEpoch(uint32_t nowMs); - // Sliding-epoch rebase: advance the epoch and shift live entries back by the - // same wall-clock amount instead of flushing, so cached state survives past the - // ~19h horizon. Caller must hold cacheLock. - void rebaseEpoch(uint32_t nowMs); + // Protected so test shims can flush per-node traffic state. + void flushCache(); + // Introspection for tests: the cached device role for a node, or -1 if the node has + // no cache entry (distinguishes "not tracked / evicted" from CLIENT == 0). + int peekCachedRole(NodeNum node); private: // ========================================================================= - // Unified Cache Entry (11 bytes) - Same for ALL platforms + // Unified Cache Entry (10 bytes) - Same for ALL platforms // ========================================================================= // - // A single compact structure used across ESP32, NRF52, and all other platforms. - // Memory: 11 bytes × TRAFFIC_MANAGEMENT_CACHE_SIZE entries (default 1000 = 11KB) - // - // Position Fingerprinting: - // Instead of storing full coordinates (8 bytes) or a computed hash, - // we store an 8-bit fingerprint derived deterministically from the - // truncated lat/lon. This extracts the lower 4 significant bits from - // each coordinate: fingerprint = (lat_low4 << 4) | lon_low4 - // - // Benefits over hash: - // - Adjacent grid cells have sequential fingerprints (no collision) - // - Two positions only collide if 16+ grid cells apart in BOTH dimensions - // - Deterministic: same input always produces same output - // - // Adaptive Timestamp Resolution: - // All timestamps use 8-bit values with adaptive resolution calculated - // from config at startup. Resolution = max(60, min(339, interval/2)). - // - Min 60 seconds ensures reasonable precision - // - Max 339 seconds allows ~24 hour range (255 * 339 = 86445 sec) - // - interval/2 ensures at least 2 ticks per configured interval - // // Layout: - // [0-3] node - NodeNum (4 bytes) - // [4] pos_fingerprint - 4 bits lat + 4 bits lon (1 byte) - // [5] rate_count - Packets in current window (1 byte) - // [6] unknown_count - Unknown packets count (1 byte) - // [7] pos_time - Position timestamp (1 byte, adaptive resolution) - // [8] rate_time - Rate window start (1 byte, adaptive resolution) - // [9] unknown_time - Unknown tracking start (1 byte, adaptive resolution) - // [10] next_hop - Last-byte relay to reach `node` (1 byte, 0 = none) + // [0-3] node - NodeNum (4 bytes, 0 = empty slot) + // [4] pos_fingerprint - 4 bits lat + 4 bits lon (0 = no position seen) + // [5] rate_count - [7:6] role[3:2] | [5:0] packets in rate window (0 = no window active) + // [6] unknown_count - [7:6] role[1:0] | [5:0] unknown packets in window (0 = no window active) + // [7] pos_time - Position tick (uint8, free-running 360 s/tick) + // [8] rate_unknown_time - [7:4] rate nibble (300 s/tick) | [3:0] unknown nibble (60 s/tick) + // [9] next_hop - Last-byte relay to reach `node` (0 = none) // - // next_hop semantics: - // A routing hint: the last byte of the NodeNum to use as next hop to reach - // `node`. Written ONLY from NextHopRouter's ACK-confirmed decision (a - // bidirectionally-verified relay), never inferred one-way from relayed - // traffic. The TMM cache acts as an overflow store for confirmed next-hops - // that have aged out of the hot NodeDB (NodeInfoLite). Unlike the other - // fields it has no TTL of its own — it keeps its slot alive (see runOnce) - // and is refreshed only on the next confirmed exchange. + // The 4-bit device role (bits [7:6] of rate_count paired with [7:6] of unknown_count) + // caches the sender's meshtastic_Config_DeviceConfig_Role as a third fallback after the + // hot store and warm store, for nodes evicted from both. Read/written via + // resolveSenderRole(). Max encodable value is 15. // + // Presence sentinels (no epoch, no +1 offset needed): + // pos active: pos_fingerprint != 0 + // rate active: getRateCount() != 0 (low 6 bits only) + // unknown active: getUnknownCount() != 0 (low 6 bits only) + // + // next_hop: routing hint written only from ACK-confirmed NextHopRouter decisions. + // No TTL — keeps the slot alive across maintenance sweeps. + // +#if _meshtastic_Config_DeviceConfig_Role_MAX > 15 +#warning "Device role enum max exceeds 15 — TMM 4-bit role cache (rate_count[7:6]/unknown_count[7:6]) will truncate new values" +#endif struct __attribute__((packed)) UnifiedCacheEntry { - NodeNum node; // 4 bytes - Node identifier (0 = empty slot) - uint8_t pos_fingerprint; // 1 byte - Lower 4 bits of lat + lon - uint8_t rate_count; // 1 byte - Packet count (saturates at 255) - uint8_t unknown_count; // 1 byte - Unknown packet count (saturates at 255) - uint8_t pos_time; // 1 byte - Position timestamp (adaptive resolution) - uint8_t rate_time; // 1 byte - Rate window start (adaptive resolution) - uint8_t unknown_time; // 1 byte - Unknown tracking start (adaptive resolution) - uint8_t next_hop; // 1 byte - Last-byte relay to reach `node` (0 = none). See note below. + NodeNum node; + uint8_t pos_fingerprint; + uint8_t rate_count; // [7:6] = role[3:2], [5:0] = count (max 63) + uint8_t unknown_count; // [7:6] = role[1:0], [5:0] = count (max 63) + uint8_t pos_time; + uint8_t rate_unknown_time; + uint8_t next_hop; + + uint8_t getRateCount() const { return rate_count & 0x3F; } + void setRateCount(uint8_t c) { rate_count = static_cast((rate_count & 0xC0) | (c & 0x3F)); } + uint8_t getUnknownCount() const { return unknown_count & 0x3F; } + void setUnknownCount(uint8_t c) { unknown_count = static_cast((unknown_count & 0xC0) | (c & 0x3F)); } + uint8_t getCachedRole() const { return static_cast(((rate_count >> 6) << 2) | (unknown_count >> 6)); } + void setCachedRole(uint8_t role) + { + rate_count = static_cast((rate_count & 0x3F) | ((role >> 2) << 6)); + unknown_count = static_cast((unknown_count & 0x3F) | ((role & 0x03) << 6)); + } + uint8_t getRateTime() const { return (rate_unknown_time >> 4) & 0x0F; } + uint8_t getUnknownTime() const { return rate_unknown_time & 0x0F; } + void setRateTime(uint8_t t) { rate_unknown_time = static_cast((rate_unknown_time & 0x0F) | ((t & 0x0F) << 4)); } + void setUnknownTime(uint8_t t) { rate_unknown_time = static_cast((rate_unknown_time & 0xF0) | (t & 0x0F)); } }; - static_assert(sizeof(UnifiedCacheEntry) == 11, "UnifiedCacheEntry should be 11 bytes"); + static_assert(sizeof(UnifiedCacheEntry) == 10, "UnifiedCacheEntry should be 10 bytes"); // ========================================================================= // Flat unified cache // ========================================================================= // // Plain array, linear scan (same idiom as WarmNodeStore). A lookup walks at - // most cacheSize() × 11 B — microseconds at LoRa packet rates, not worth a + // most cacheSize() × 10 B — microseconds at LoRa packet rates, not worth a // hash table. Insertion on a full cache evicts the stalest entry, // preferring entries without a next_hop hint (those are the long-tail // routing state this cache exists to keep). @@ -154,82 +166,29 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread static constexpr uint16_t nodeInfoTargetEntries() { return kNodeInfoCacheEntries; } // ========================================================================= - // Adaptive Timestamp Resolution + // Free-Running Tick Counters // ========================================================================= // - // All timestamps use 8-bit values with adaptive resolution calculated from - // config at startup. This allows ~24 hour range while maintaining precision. + // Timestamps are stored as free-running modular tick counters derived from + // millis(). No epoch anchor needed: modular subtraction gives correct age + // as long as the true age stays below the counter period. // - // Resolution formula: max(60, min(339, interval/2)) - // - 60 sec minimum ensures reasonable precision - // - 339 sec maximum allows 24 hour range (255 * 339 ≈ 86400 sec) - // - interval/2 ensures at least 2 ticks per configured interval + // pos_time : uint8 (256 ticks × 360 s = 25.6 h period; max window 12 h = 120 ticks) + // rate_time : nibble (16 ticks × 300 s = 80 min period; max window 1 h = 12 ticks) + // unknown_time: nibble (16 ticks × 60 s = 16 min period; max window 12 min = 12 ticks) // - // Since config changes require reboot, resolution is calculated once. + // Presence sentinels (no +1 offset needed; count fields serve as guards): + // pos active: pos_fingerprint != 0 (0 is reserved sentinel; computePositionFingerprint() remaps computed-0 → 0xFF) + // rate active: getRateCount() != 0 (low 6 bits; high 2 bits are cached role) + // unknown active: getUnknownCount() != 0 // - uint32_t cacheEpochMs = 0; - uint16_t posTimeResolution = 60; // Seconds per tick for position - uint16_t rateTimeResolution = 60; // Seconds per tick for rate limiting - uint16_t unknownTimeResolution = 60; // Seconds per tick for unknown tracking + static constexpr uint32_t kPosTimeTickMs = 360'000UL; // 6 min/tick + static constexpr uint32_t kRateTimeTickMs = 300'000UL; // 5 min/tick + static constexpr uint32_t kUnknownTimeTickMs = 60'000UL; // 1 min/tick - // Calculate resolution from configured interval (called once at startup) - static uint16_t calcTimeResolution(uint32_t intervalSecs) - { - // Resolution = interval/2 to ensure at least 2 ticks per interval - // Clamped to [60, 339] for min precision and max 24h range - uint32_t res = (intervalSecs > 0) ? (intervalSecs / 2) : 60; - if (res < 60) - res = 60; - if (res > 339) - res = 339; - return static_cast(res); - } - - // Convert to/from 8-bit relative timestamps with given resolution. - // - // All stored timestamps carry a uniform +1 "presence" offset: a value of 0 is - // reserved for "no timestamp recorded" (which is also the zero-initialized - // state), and stored values 1..255 encode raw ticks 0..254. This keeps the - // 0-means-empty sentinel consistent with memset/calloc zeroing across every - // sub-store, so the maintenance sweep's `_time != 0` presence checks are - // unambiguous (a timestamp recorded in the first tick after the epoch is no - // longer mistaken for an empty slot). The offset is applied here and removed - // on read, so it cancels out in all window math. - uint8_t toRelativeTime(uint32_t nowMs, uint16_t resolutionSecs) const - { - uint32_t ticks = (nowMs - cacheEpochMs) / (resolutionSecs * 1000UL); - return (ticks >= UINT8_MAX) ? UINT8_MAX : static_cast(ticks + 1); - } - uint32_t fromRelativeTime(uint8_t ticks, uint16_t resolutionSecs) const - { - return (ticks == 0) ? cacheEpochMs : cacheEpochMs + (static_cast(ticks - 1) * resolutionSecs * 1000UL); - } - - // Convenience wrappers for each timestamp type (the +1 presence offset lives - // in the shared converters above, so these are plain pass-throughs). - uint8_t toRelativePosTime(uint32_t nowMs) const { return toRelativeTime(nowMs, posTimeResolution); } - uint32_t fromRelativePosTime(uint8_t t) const { return fromRelativeTime(t, posTimeResolution); } - - uint8_t toRelativeRateTime(uint32_t nowMs) const { return toRelativeTime(nowMs, rateTimeResolution); } - uint32_t fromRelativeRateTime(uint8_t t) const { return fromRelativeTime(t, rateTimeResolution); } - - uint8_t toRelativeUnknownTime(uint32_t nowMs) const { return toRelativeTime(nowMs, unknownTimeResolution); } - uint32_t fromRelativeUnknownTime(uint8_t t) const { return fromRelativeTime(t, unknownTimeResolution); } - - // Coarsest of the per-feature resolutions (seconds per tick). - uint16_t maxResolution() const - { - uint16_t maxRes = posTimeResolution; - if (rateTimeResolution > maxRes) - maxRes = rateTimeResolution; - if (unknownTimeResolution > maxRes) - maxRes = unknownTimeResolution; - return maxRes; - } - - // True when relative offsets approach 8-bit overflow. - // With max resolution of 339 sec, 200 ticks = ~19 hours (safe margin for 24h max). - bool needsEpochReset(uint32_t nowMs) const { return (nowMs - cacheEpochMs) > (200UL * maxResolution() * 1000UL); } + static uint8_t currentPosTick() { return static_cast(clockMs() / kPosTimeTickMs); } + static uint8_t currentRateTick() { return static_cast((clockMs() / kRateTimeTickMs) & 0x0F); } + static uint8_t currentUnknownTick() { return static_cast((clockMs() / kUnknownTimeTickMs) & 0x0F); } // ========================================================================= // Position Fingerprint // ========================================================================= @@ -309,6 +268,21 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread // Find existing entry (no creation) UnifiedCacheEntry *findEntry(NodeNum node); + // Resolve a sender's advertised device role for the position hot path. The tier-3 + // cache (this entry's getCachedRole) is authoritative and is kept fresh by + // updateCachedRoleFromNodeInfo() — updated when NodeDB learns a role, not re-derived + // per packet. Only on first tracking (isNew) do we scan NodeDB (hot store → warm + // store, via getNodeRole) to seed the cache, so a resident special-role node is + // correct from its first position; after that the read is O(1) and survives the node + // aging out of both NodeDB stores. Caller must hold cacheLock; entry may be null + // (→ NodeDB scan only). + meshtastic_Config_DeviceConfig_Role resolveSenderRole(NodeNum from, UnifiedCacheEntry *entry, bool isNew); + + // Refresh the tier-3 role cache from an observed NodeInfo (the same event that updates + // NodeDB's role). Reads role from the packet's User payload; updates only nodes already + // tracked (no entry creation). Takes cacheLock. + void updateCachedRoleFromNodeInfo(const meshtastic_MeshPacket &mp); + // NodeInfo cache operations (flat PSRAM payload array, linear scan) const NodeInfoPayloadEntry *findNodeInfoEntry(NodeNum node) const; NodeInfoPayloadEntry *findOrCreateNodeInfoEntry(NodeNum node, bool *usedEmptySlot); diff --git a/test/test_position_module/test_main.cpp b/test/test_position_module/test_main.cpp new file mode 100644 index 000000000..778779af6 --- /dev/null +++ b/test/test_position_module/test_main.cpp @@ -0,0 +1,81 @@ +#include "TestUtil.h" +#include "modules/PositionModule.h" +#include + +// These exercise PositionModule's pure broadcast-policy helpers (stationary detection and the +// interval floor). They take plain values, so no device globals or fake clock are needed. + +// Coordinates sharing the top `precision` bits land in the same grid cell. +static void test_withinPrecisionCell_jitterStaysInCell() +{ + // At precision 16 the top 16 bits define the cell; the low 16 bits are GPS jitter. + TEST_ASSERT_TRUE(PositionModule::positionWithinPrecisionCell(0x12340000, 0x22340000, 0x1234ABCD, 0x2234EF01, 16)); +} + +static void test_withinPrecisionCell_movingLatLeavesCell() +{ + TEST_ASSERT_FALSE(PositionModule::positionWithinPrecisionCell(0x12340000, 0x22340000, 0x12350000, 0x22340000, 16)); +} + +static void test_withinPrecisionCell_movingLonLeavesCell() +{ + TEST_ASSERT_FALSE(PositionModule::positionWithinPrecisionCell(0x12340000, 0x22340000, 0x12340000, 0x22350000, 16)); +} + +// precision 0 means position sharing is off — never treat as stationary/suppressible. +static void test_withinPrecisionCell_zeroPrecisionNeverSuppresses() +{ + TEST_ASSERT_FALSE(PositionModule::positionWithinPrecisionCell(0x12340000, 0x22340000, 0x12340000, 0x22340000, 0)); +} + +// Full precision (>=32): any difference matters, and identical full-precision coords still aren't +// "stationary" because there's no coarse cell to hold within. +static void test_withinPrecisionCell_fullPrecisionNeverSuppresses() +{ + TEST_ASSERT_FALSE(PositionModule::positionWithinPrecisionCell(0x12340000, 0x22340000, 0x12340000, 0x22340000, 32)); +} + +static void test_effectiveInterval_stationaryRaisesToFloor() +{ + TEST_ASSERT_EQUAL_UINT32(43200000U, PositionModule::effectiveBroadcastIntervalMs(60000U, true, 43200000U)); +} + +static void test_effectiveInterval_movingKeepsConfigured() +{ + TEST_ASSERT_EQUAL_UINT32(60000U, PositionModule::effectiveBroadcastIntervalMs(60000U, false, 43200000U)); +} + +// A configured interval already longer than the floor is never shortened. +static void test_effectiveInterval_longConfiguredWinsOverFloor() +{ + TEST_ASSERT_EQUAL_UINT32(50000000U, PositionModule::effectiveBroadcastIntervalMs(50000000U, true, 43200000U)); +} + +static void test_effectiveInterval_zeroFloorIsNoOp() +{ + TEST_ASSERT_EQUAL_UINT32(60000U, PositionModule::effectiveBroadcastIntervalMs(60000U, true, 0U)); +} + +void setUp(void) {} + +void tearDown(void) {} + +extern "C" { +void setup() +{ + initializeTestEnvironment(); + UNITY_BEGIN(); + RUN_TEST(test_withinPrecisionCell_jitterStaysInCell); + RUN_TEST(test_withinPrecisionCell_movingLatLeavesCell); + RUN_TEST(test_withinPrecisionCell_movingLonLeavesCell); + RUN_TEST(test_withinPrecisionCell_zeroPrecisionNeverSuppresses); + RUN_TEST(test_withinPrecisionCell_fullPrecisionNeverSuppresses); + RUN_TEST(test_effectiveInterval_stationaryRaisesToFloor); + RUN_TEST(test_effectiveInterval_movingKeepsConfigured); + RUN_TEST(test_effectiveInterval_longConfiguredWinsOverFloor); + RUN_TEST(test_effectiveInterval_zeroFloorIsNoOp); + exit(UNITY_END()); +} + +void loop() {} +} diff --git a/test/test_traffic_management/test_main.cpp b/test/test_traffic_management/test_main.cpp index 7f314ae4b..1844653ab 100644 --- a/test/test_traffic_management/test_main.cpp +++ b/test/test_traffic_management/test_main.cpp @@ -13,6 +13,7 @@ #include "airtime.h" #include "mesh/CryptoEngine.h" +#include "mesh/Default.h" #include "mesh/MeshService.h" #include "mesh/NodeDB.h" #include "mesh/Router.h" @@ -78,6 +79,13 @@ class MockNodeDB : public NodeDB // Role the TMM should see for the cached node (sender-role-aware throttles). void setCachedNodeRole(meshtastic_Config_DeviceConfig_Role role) { cachedNode.role = role; } + // Direct mutable access to the cached node for fine-grained bitfield manipulation in tests. + meshtastic_NodeInfoLite &cachedNodeForTest() + { + hasCachedNode = true; + return cachedNode; + } + // Seed a node into the hot-store buffer at index 1 (index 0 is reserved for // "self"). Respects the fixed-buffer invariant: `meshNodes` is a buffer of // MAX_NUM_NODES slots with `numMeshNodes` as the logical count — we grow the @@ -149,8 +157,9 @@ class TrafficManagementModuleTestShim : public TrafficManagementModule { public: using TrafficManagementModule::alterReceived; + using TrafficManagementModule::flushCache; using TrafficManagementModule::handleReceived; - using TrafficManagementModule::resetEpoch; + using TrafficManagementModule::peekCachedRole; using TrafficManagementModule::runOnce; bool ignoreRequestFlag() const { return ignoreRequest; } @@ -163,7 +172,6 @@ static void resetTrafficConfig() moduleConfig = meshtastic_LocalModuleConfig_init_zero; moduleConfig.has_traffic_management = true; moduleConfig.traffic_management = meshtastic_ModuleConfig_TrafficManagementConfig_init_zero; - moduleConfig.traffic_management.enabled = true; config = meshtastic_LocalConfig_init_zero; config.device.role = meshtastic_Config_DeviceConfig_Role_CLIENT; @@ -179,6 +187,10 @@ static void resetTrafficConfig() mockNodeDB->resetNodes(); mockNodeDB->clearCachedNode(); nodeDB = mockNodeDB; + + // Virtual clock base (1 h in, so tick subtraction never underflows). Tests advance time by + // bumping TrafficManagementModule::s_testNowMs instead of sleeping real seconds across a tick. + TrafficManagementModule::s_testNowMs = 3600000; } static meshtastic_MeshPacket makeDecodedPacket(meshtastic_PortNum port, NodeNum from, NodeNum to = NODENUM_BROADCAST) @@ -261,6 +273,16 @@ static void installWellKnownPrimaryChannel() config.lora.modem_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_FAST; } +// Install the well-known primary channel AND set a specific position_precision so +// shouldDropPosition() uses that precision ceiling rather than the default fallback. +// precision=0 means "no channel ceiling" and falls back to the firmware default (19 bits). +static void installWellKnownPrimaryChannelWithPrecision(uint32_t precision) +{ + installWellKnownPrimaryChannel(); + channelFile.channels[0].settings.has_module_settings = true; + channelFile.channels[0].settings.module_settings.position_precision = precision; +} + static meshtastic_MeshPacket makeNodeInfoPacket(NodeNum from, const char *longName, const char *shortName) { meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_NODEINFO_APP, from, NODENUM_BROADCAST); @@ -275,6 +297,21 @@ static meshtastic_MeshPacket makeNodeInfoPacket(NodeNum from, const char *longNa return packet; } +static meshtastic_MeshPacket makeNodeInfoPacketWithRole(NodeNum from, meshtastic_Config_DeviceConfig_Role role) +{ + meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_NODEINFO_APP, from, NODENUM_BROADCAST); + + meshtastic_User user = meshtastic_User_init_zero; + snprintf(user.id, sizeof(user.id), "!%08x", from); + strncpy(user.long_name, "rolenode", sizeof(user.long_name) - 1); + strncpy(user.short_name, "rn", sizeof(user.short_name) - 1); + user.role = role; + + packet.decoded.payload.size = + pb_encode_to_bytes(packet.decoded.payload.bytes, sizeof(packet.decoded.payload.bytes), &meshtastic_User_msg, &user); + return packet; +} + /** * Verify the module is a no-op when traffic management is disabled. * Important so config toggles cannot accidentally change routing behavior. @@ -300,7 +337,6 @@ static void test_tm_moduleDisabled_doesNothing(void) */ static void test_tm_unknownPackets_dropOnNPlusOne(void) { - moduleConfig.traffic_management.drop_unknown_enabled = true; moduleConfig.traffic_management.unknown_packet_threshold = 2; TrafficManagementModuleTestShim module; meshtastic_MeshPacket packet = makeUnknownPacket(kRemoteNode); @@ -324,9 +360,8 @@ static void test_tm_unknownPackets_dropOnNPlusOne(void) */ static void test_tm_positionDedup_dropsDuplicateWithinWindow(void) { - moduleConfig.traffic_management.position_dedup_enabled = true; - moduleConfig.traffic_management.position_precision_bits = 16; moduleConfig.traffic_management.position_min_interval_secs = 300; + installWellKnownPrimaryChannelWithPrecision(16); TrafficManagementModuleTestShim module; meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); @@ -349,9 +384,8 @@ static void test_tm_positionDedup_dropsDuplicateWithinWindow(void) */ static void test_tm_positionDedup_allowsMovedPosition(void) { - moduleConfig.traffic_management.position_dedup_enabled = true; - moduleConfig.traffic_management.position_precision_bits = 16; moduleConfig.traffic_management.position_min_interval_secs = 300; + installWellKnownPrimaryChannelWithPrecision(16); TrafficManagementModuleTestShim module; meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); @@ -372,7 +406,6 @@ static void test_tm_positionDedup_allowsMovedPosition(void) */ static void test_tm_rateLimit_dropsOnlyAfterThreshold(void) { - moduleConfig.traffic_management.rate_limit_enabled = true; moduleConfig.traffic_management.rate_limit_window_secs = 60; moduleConfig.traffic_management.rate_limit_max_packets = 3; TrafficManagementModuleTestShim module; @@ -397,12 +430,10 @@ static void test_tm_rateLimit_dropsOnlyAfterThreshold(void) */ static void test_tm_fromUs_bypassesPositionAndRateFilters(void) { - moduleConfig.traffic_management.position_dedup_enabled = true; - moduleConfig.traffic_management.position_precision_bits = 16; moduleConfig.traffic_management.position_min_interval_secs = 300; - moduleConfig.traffic_management.rate_limit_enabled = true; moduleConfig.traffic_management.rate_limit_window_secs = 60; moduleConfig.traffic_management.rate_limit_max_packets = 1; + installWellKnownPrimaryChannelWithPrecision(16); TrafficManagementModuleTestShim module; meshtastic_MeshPacket positionPacket = makePositionPacket(kLocalNode, 374221234, -1220845678); @@ -428,12 +459,10 @@ static void test_tm_fromUs_bypassesPositionAndRateFilters(void) */ static void test_tm_localDestination_bypassesTransitFilters(void) { - moduleConfig.traffic_management.position_dedup_enabled = true; - moduleConfig.traffic_management.position_precision_bits = 16; moduleConfig.traffic_management.position_min_interval_secs = 300; - moduleConfig.traffic_management.rate_limit_enabled = true; moduleConfig.traffic_management.rate_limit_window_secs = 60; moduleConfig.traffic_management.rate_limit_max_packets = 1; + installWellKnownPrimaryChannelWithPrecision(16); TrafficManagementModuleTestShim module; meshtastic_MeshPacket position1 = makePositionPacket(kRemoteNode, 374221234, -1220845678, kLocalNode); @@ -461,7 +490,6 @@ static void test_tm_localDestination_bypassesTransitFilters(void) */ static void test_tm_nodeinfo_routerClamp_skipsWhenTooManyHops(void) { - moduleConfig.traffic_management.nodeinfo_direct_response = true; moduleConfig.traffic_management.nodeinfo_direct_response_max_hops = 10; config.device.role = meshtastic_Config_DeviceConfig_Role_ROUTER; mockNodeDB->setCachedNode(kTargetNode); @@ -486,7 +514,6 @@ static void test_tm_nodeinfo_routerClamp_skipsWhenTooManyHops(void) */ static void test_tm_nodeinfo_directResponse_respondsFromCache(void) { - moduleConfig.traffic_management.nodeinfo_direct_response = true; moduleConfig.traffic_management.nodeinfo_direct_response_max_hops = 10; config.device.role = meshtastic_Config_DeviceConfig_Role_CLIENT; config.lora.config_ok_to_mqtt = true; @@ -532,7 +559,6 @@ static void test_tm_nodeinfo_directResponse_respondsFromCache(void) */ static void test_tm_nodeinfo_directResponse_learnsRequestorNodeInfo(void) { - moduleConfig.traffic_management.nodeinfo_direct_response = true; moduleConfig.traffic_management.nodeinfo_direct_response_max_hops = 10; config.device.role = meshtastic_Config_DeviceConfig_Role_CLIENT; mockNodeDB->setCachedNode(kTargetNode); @@ -568,7 +594,6 @@ static void test_tm_nodeinfo_directResponse_learnsRequestorNodeInfo(void) */ static void test_tm_nodeinfo_clientClamp_skipsWhenNotDirect(void) { - moduleConfig.traffic_management.nodeinfo_direct_response = true; moduleConfig.traffic_management.nodeinfo_direct_response_max_hops = 10; config.device.role = meshtastic_Config_DeviceConfig_Role_CLIENT; mockNodeDB->setCachedNode(kTargetNode); @@ -595,7 +620,6 @@ static void test_tm_nodeinfo_clientClamp_skipsWhenNotDirect(void) */ static void test_tm_nodeinfo_directResponse_withoutNodeDbEntry_skips(void) { - moduleConfig.traffic_management.nodeinfo_direct_response = true; moduleConfig.traffic_management.nodeinfo_direct_response_max_hops = 10; config.device.role = meshtastic_Config_DeviceConfig_Role_CLIENT; mockNodeDB->clearCachedNode(); @@ -629,7 +653,6 @@ static void test_tm_nodeinfo_directResponse_withoutNodeDbEntry_skips(void) */ static void test_tm_nodeinfo_directResponse_psramCacheRespondsAndPreservesBitfield(void) { - moduleConfig.traffic_management.nodeinfo_direct_response = true; moduleConfig.traffic_management.nodeinfo_direct_response_max_hops = 10; config.device.role = meshtastic_Config_DeviceConfig_Role_CLIENT; config.lora.config_ok_to_mqtt = true; @@ -683,7 +706,6 @@ static void test_tm_nodeinfo_directResponse_psramCacheRespondsAndPreservesBitfie */ static void test_tm_nodeinfo_directResponse_psramMissDoesNotFallbackToNodeDb(void) { - moduleConfig.traffic_management.nodeinfo_direct_response = true; moduleConfig.traffic_management.nodeinfo_direct_response_max_hops = 10; config.device.role = meshtastic_Config_DeviceConfig_Role_CLIENT; mockNodeDB->setCachedNode(kTargetNode); @@ -711,15 +733,13 @@ static void test_tm_nodeinfo_directResponse_psramMissDoesNotFallbackToNodeDb(voi #endif /** - * Verify relayed telemetry broadcasts are hop-exhausted when enabled AND the - * channel is congested (telemetry exhaustion is gated on channel utilization, - * unlike position exhaustion). - * Important to prevent further mesh propagation while still allowing one relay step. + * Verify relayed telemetry broadcasts are NOT hop-exhausted. + * exhaust_hop_telemetry / exhaust_hop_position have been removed from the config + * as "not suitable right now" — alterReceived must leave hop_limit unchanged. */ -static void test_tm_alterReceived_exhaustsRelayedTelemetryBroadcast(void) +static void test_tm_alterReceived_telemetryBroadcast_hopLimitUnchanged(void) { - moduleConfig.traffic_management.exhaust_hop_telemetry = true; - ScopedBusyAirTime busyChannel; + ScopedBusyAirTime busyChannel; // congestion present but exhaust is disabled TrafficManagementModuleTestShim module; meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode, NODENUM_BROADCAST); packet.hop_start = 5; @@ -728,20 +748,19 @@ static void test_tm_alterReceived_exhaustsRelayedTelemetryBroadcast(void) module.alterReceived(packet); meshtastic_TrafficManagementStats stats = module.getStats(); - TEST_ASSERT_EQUAL_UINT8(0, packet.hop_limit); - TEST_ASSERT_EQUAL_UINT8(3, packet.hop_start); - TEST_ASSERT_TRUE(module.shouldExhaustHops(packet)); - TEST_ASSERT_EQUAL_UINT32(1, stats.hop_exhausted_packets); + TEST_ASSERT_EQUAL_UINT8(3, packet.hop_limit); // unchanged + TEST_ASSERT_EQUAL_UINT8(5, packet.hop_start); // unchanged + TEST_ASSERT_FALSE(module.shouldExhaustHops(packet)); + TEST_ASSERT_EQUAL_UINT32(0, stats.hop_exhausted_packets); } /** - * Verify hop exhaustion skips unicast and local-origin packets. - * Important to avoid mutating traffic that should retain normal forwarding behavior. + * Verify alterReceived does not modify unicast or local-origin packets. + * The precision clamp (the only active alterReceived path) only fires for + * broadcast position packets from remote nodes — these should be untouched. */ static void test_tm_alterReceived_skipsLocalAndUnicast(void) { - moduleConfig.traffic_management.exhaust_hop_telemetry = true; - ScopedBusyAirTime busyChannel; // congestion satisfied, so only the skip conditions are under test TrafficManagementModuleTestShim module; meshtastic_MeshPacket unicast = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode, kTargetNode); @@ -768,9 +787,9 @@ static void test_tm_alterReceived_skipsLocalAndUnicast(void) */ static void test_tm_positionDedup_allowsDuplicateAfterIntervalExpires(void) { - moduleConfig.traffic_management.position_dedup_enabled = true; - moduleConfig.traffic_management.position_precision_bits = 16; - moduleConfig.traffic_management.position_min_interval_secs = 1; + // 360 s = 1 pos-tick (kPosTimeTickMs); advance the virtual clock past one tick period. + moduleConfig.traffic_management.position_min_interval_secs = 360; + installWellKnownPrimaryChannelWithPrecision(16); TrafficManagementModuleTestShim module; meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); @@ -779,7 +798,7 @@ static void test_tm_positionDedup_allowsDuplicateAfterIntervalExpires(void) ProcessMessage r1 = module.handleReceived(first); ProcessMessage r2 = module.handleReceived(second); - testDelay(1200); + TrafficManagementModule::s_testNowMs += 360001; // advance past one 6-min pos-tick (virtual clock) ProcessMessage r3 = module.handleReceived(third); meshtastic_TrafficManagementStats stats = module.getStats(); @@ -795,9 +814,9 @@ static void test_tm_positionDedup_allowsDuplicateAfterIntervalExpires(void) */ static void test_tm_positionDedup_intervalZero_neverDrops(void) { - moduleConfig.traffic_management.position_dedup_enabled = true; - moduleConfig.traffic_management.position_precision_bits = 16; + // position_min_interval_secs=0 disables the drop gate (shouldDropPosition returns false for any packet). moduleConfig.traffic_management.position_min_interval_secs = 0; + installWellKnownPrimaryChannelWithPrecision(16); TrafficManagementModuleTestShim module; meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); @@ -818,9 +837,9 @@ static void test_tm_positionDedup_intervalZero_neverDrops(void) */ static void test_tm_positionDedup_precisionAbove32_usesDefaultPrecision(void) { - moduleConfig.traffic_management.position_dedup_enabled = true; - moduleConfig.traffic_management.position_precision_bits = 99; + // Channel precision=99 is out of range; sanitizePositionPrecision falls back to default (19 bits). moduleConfig.traffic_management.position_min_interval_secs = 300; + installWellKnownPrimaryChannelWithPrecision(99); TrafficManagementModuleTestShim module; meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); @@ -836,18 +855,21 @@ static void test_tm_positionDedup_precisionAbove32_usesDefaultPrecision(void) } /** - * Verify precision=32 does not collapse all positions to one fingerprint. - * Important to prevent false duplicate drops at the full-precision boundary. + * Verify the dedup fingerprint does not collapse positions that are distinct at the + * channel's *effective* precision. Dedup only runs on well-known (public) channels, + * where precision is capped at MAX_POSITION_PRECISION_PUBLIC_KEY (15) regardless of the + * channel's configured value — so the requested 32 is clamped to 15. Positions must + * therefore differ in the top 15 bits (>= 2^17 raw units) to read as distinct; here + * they differ by 2^18, well clear of the precision-15 grid, so neither is dropped. */ -static void test_tm_positionDedup_precision32_allowsDistinctPositions(void) +static void test_tm_positionDedup_distinctAtClampedChannelPrecision(void) { - moduleConfig.traffic_management.position_dedup_enabled = true; - moduleConfig.traffic_management.position_precision_bits = 32; moduleConfig.traffic_management.position_min_interval_secs = 300; + installWellKnownPrimaryChannelWithPrecision(32); // clamped to 15 on a public channel TrafficManagementModuleTestShim module; meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); - meshtastic_MeshPacket second = makePositionPacket(kRemoteNode, 374221235, -1220845677); + meshtastic_MeshPacket second = makePositionPacket(kRemoteNode, 374221234 + (1 << 18), -1220845678 + (1 << 18)); ProcessMessage r1 = module.handleReceived(first); ProcessMessage r2 = module.handleReceived(second); @@ -859,17 +881,15 @@ static void test_tm_positionDedup_precision32_allowsDistinctPositions(void) } /** - * Verify precision=0 falls back to the default precision (same contract as - * >32: getConfiguredOrDefault + sanitizePositionPrecision treat 0 as unset). - * Important so invalid config does not collapse all positions into one - * fingerprint — positions in different default-precision grid cells must - * still be distinct. + * Verify channel precision=0 (no channel ceiling set) falls back to the firmware + * default precision (19 bits / ~90 m cells). Positions more than one default grid + * cell apart must remain distinct, not collapse into one fingerprint. */ -static void test_tm_positionDedup_precisionZero_allowsDistinctPositions(void) +static void test_tm_positionDedup_precisionZero_channelFallsBackToDefault(void) { - moduleConfig.traffic_management.position_dedup_enabled = true; - moduleConfig.traffic_management.position_precision_bits = 0; moduleConfig.traffic_management.position_min_interval_secs = 300; + // precision=0 in the channel → getPositionPrecisionForChannel returns 0 → falls back to default 19 bits. + installWellKnownPrimaryChannelWithPrecision(0); TrafficManagementModuleTestShim module; meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); @@ -888,20 +908,19 @@ static void test_tm_positionDedup_precisionZero_allowsDistinctPositions(void) * Verify epoch reset invalidates stale position identity for dedup. * Important so reset paths cannot leak prior packet identity into new windows. */ -static void test_tm_positionDedup_epochReset_doesNotDropFirstPacketAfterReset(void) +static void test_tm_positionDedup_cacheFlush_doesNotDropFirstPacketAfterFlush(void) { - moduleConfig.traffic_management.position_dedup_enabled = true; - moduleConfig.traffic_management.position_precision_bits = 16; moduleConfig.traffic_management.position_min_interval_secs = 300; + installWellKnownPrimaryChannelWithPrecision(16); TrafficManagementModuleTestShim module; meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); - meshtastic_MeshPacket afterReset = makePositionPacket(kRemoteNode, 374221234, -1220845678); + meshtastic_MeshPacket afterFlush = makePositionPacket(kRemoteNode, 374221234, -1220845678); meshtastic_MeshPacket duplicate = makePositionPacket(kRemoteNode, 374221234, -1220845678); ProcessMessage r1 = module.handleReceived(first); - module.resetEpoch(millis()); - ProcessMessage r2 = module.handleReceived(afterReset); + module.flushCache(); + ProcessMessage r2 = module.handleReceived(afterFlush); ProcessMessage r3 = module.handleReceived(duplicate); meshtastic_TrafficManagementStats stats = module.getStats(); @@ -917,12 +936,10 @@ static void test_tm_positionDedup_epochReset_doesNotDropFirstPacketAfterReset(vo */ static void test_tm_positionDedup_priorRateState_doesNotDropFirstFingerprintZero(void) { - moduleConfig.traffic_management.position_dedup_enabled = true; - moduleConfig.traffic_management.position_precision_bits = 16; moduleConfig.traffic_management.position_min_interval_secs = 300; - moduleConfig.traffic_management.rate_limit_enabled = true; moduleConfig.traffic_management.rate_limit_window_secs = 60; moduleConfig.traffic_management.rate_limit_max_packets = 10; + installWellKnownPrimaryChannelWithPrecision(16); TrafficManagementModuleTestShim module; meshtastic_MeshPacket telemetry = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode); @@ -946,15 +963,15 @@ static void test_tm_positionDedup_priorRateState_doesNotDropFirstFingerprintZero */ static void test_tm_rateLimit_resetsAfterWindowExpires(void) { - moduleConfig.traffic_management.rate_limit_enabled = true; - moduleConfig.traffic_management.rate_limit_window_secs = 1; + // 300 s = 1 rate-tick (kRateTimeTickMs); advance the virtual clock past one tick period. + moduleConfig.traffic_management.rate_limit_window_secs = 300; moduleConfig.traffic_management.rate_limit_max_packets = 1; TrafficManagementModuleTestShim module; meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode); ProcessMessage r1 = module.handleReceived(packet); ProcessMessage r2 = module.handleReceived(packet); - testDelay(1200); + TrafficManagementModule::s_testNowMs += 300001; // advance past one 5-min rate-tick (virtual clock) ProcessMessage r3 = module.handleReceived(packet); meshtastic_TrafficManagementStats stats = module.getStats(); @@ -969,15 +986,13 @@ static void test_tm_rateLimit_resetsAfterWindowExpires(void) */ static void test_tm_unknownPackets_resetAfterWindowExpires(void) { - moduleConfig.traffic_management.drop_unknown_enabled = true; moduleConfig.traffic_management.unknown_packet_threshold = 1; - moduleConfig.traffic_management.rate_limit_window_secs = 1; TrafficManagementModuleTestShim module; meshtastic_MeshPacket packet = makeUnknownPacket(kRemoteNode); ProcessMessage r1 = module.handleReceived(packet); ProcessMessage r2 = module.handleReceived(packet); - testDelay(1200); + TrafficManagementModule::s_testNowMs += 300001; // advance past 5 unknown-ticks (5 × 60s) (virtual clock) ProcessMessage r3 = module.handleReceived(packet); meshtastic_TrafficManagementStats stats = module.getStats(); @@ -988,17 +1003,17 @@ static void test_tm_unknownPackets_resetAfterWindowExpires(void) } /** - * Verify unknown threshold values above 255 clamp to the counter ceiling. - * Important to align config semantics with saturating counter storage. + * Verify unknown threshold values above the implementation cap (60) are clamped. + * The counter is a 6-bit field (saturates at 63); threshold is capped at 60 so a + * saturated reading always exceeds it. A config value of 300 should behave as 60. */ static void test_tm_unknownPackets_thresholdAbove255_clamps(void) { - moduleConfig.traffic_management.drop_unknown_enabled = true; moduleConfig.traffic_management.unknown_packet_threshold = 300; TrafficManagementModuleTestShim module; meshtastic_MeshPacket packet = makeUnknownPacket(kRemoteNode); - for (int i = 0; i < 255; i++) { + for (int i = 0; i < 60; i++) { ProcessMessage result = module.handleReceived(packet); TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(result)); } @@ -1010,14 +1025,11 @@ static void test_tm_unknownPackets_thresholdAbove255_clamps(void) } /** - * Verify relayed position broadcasts can also be hop-exhausted — under the - * same pressure gate as telemetry (here: channel congestion). - * Important because telemetry and position use separate exhaust flags. + * Verify relayed position broadcasts are NOT hop-exhausted. + * exhaust_hop_position has been removed — alterReceived must leave hop_limit unchanged. */ -static void test_tm_alterReceived_exhaustsRelayedPositionBroadcast(void) +static void test_tm_alterReceived_positionBroadcast_hopLimitUnchanged(void) { - moduleConfig.traffic_management.exhaust_hop_position = true; - ScopedBusyAirTime busyChannel; TrafficManagementModuleTestShim module; meshtastic_MeshPacket packet = makePositionPacket(kRemoteNode, 374221234, -1220845678, NODENUM_BROADCAST); packet.hop_start = 5; @@ -1026,19 +1038,17 @@ static void test_tm_alterReceived_exhaustsRelayedPositionBroadcast(void) module.alterReceived(packet); meshtastic_TrafficManagementStats stats = module.getStats(); - TEST_ASSERT_EQUAL_UINT8(0, packet.hop_limit); - TEST_ASSERT_EQUAL_UINT8(4, packet.hop_start); - TEST_ASSERT_TRUE(module.shouldExhaustHops(packet)); - TEST_ASSERT_EQUAL_UINT32(1, stats.hop_exhausted_packets); + TEST_ASSERT_EQUAL_UINT8(2, packet.hop_limit); // unchanged + TEST_ASSERT_EQUAL_UINT8(5, packet.hop_start); // unchanged + TEST_ASSERT_FALSE(module.shouldExhaustHops(packet)); + TEST_ASSERT_EQUAL_UINT32(0, stats.hop_exhausted_packets); } /** - * Verify hop exhaustion ignores undecoded/encrypted packets. + * Verify alterReceived ignores undecoded/encrypted packets. * Important so we never mutate packets that were not decoded by this module. */ static void test_tm_alterReceived_skipsUndecodedPackets(void) { - moduleConfig.traffic_management.exhaust_hop_telemetry = true; - ScopedBusyAirTime busyChannel; // congestion satisfied, so only the undecoded skip is under test TrafficManagementModuleTestShim module; meshtastic_MeshPacket packet = makeUnknownPacket(kRemoteNode, NODENUM_BROADCAST); packet.hop_start = 5; @@ -1054,20 +1064,19 @@ static void test_tm_alterReceived_skipsUndecodedPackets(void) } /** - * Verify exhaustRequested is per-packet and resets on next handleReceived(). - * Important so a prior packet cannot leak hop-exhaust state into later packets. + * Verify shouldExhaustHops() always returns false — exhaust_hop_* features are + * removed, so the exhaustRequested flag is never set. + * Guards against accidental re-enablement without updating the flag logic. */ -static void test_tm_alterReceived_resetExhaustFlagOnNextPacket(void) +static void test_tm_alterReceived_exhaustFlagAlwaysFalse(void) { - moduleConfig.traffic_management.exhaust_hop_telemetry = true; - ScopedBusyAirTime busyChannel; // telemetry exhaust only fires under congestion TrafficManagementModuleTestShim module; meshtastic_MeshPacket telemetry = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode, NODENUM_BROADCAST); telemetry.hop_start = 5; telemetry.hop_limit = 3; module.alterReceived(telemetry); - TEST_ASSERT_TRUE(module.shouldExhaustHops(telemetry)); + TEST_ASSERT_FALSE(module.shouldExhaustHops(telemetry)); meshtastic_MeshPacket text = makeDecodedPacket(meshtastic_PortNum_TEXT_MESSAGE_APP, kRemoteNode); ProcessMessage result = module.handleReceived(text); @@ -1075,42 +1084,40 @@ static void test_tm_alterReceived_resetExhaustFlagOnNextPacket(void) TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(result)); TEST_ASSERT_FALSE(module.shouldExhaustHops(telemetry)); - TEST_ASSERT_EQUAL_UINT32(1, stats.hop_exhausted_packets); + TEST_ASSERT_EQUAL_UINT32(0, stats.hop_exhausted_packets); } /** - * Verify exhaust requests are packet-scoped (from + id). - * Important so stale state from one packet cannot influence unrelated packets - * that pass through duplicate/rebroadcast paths before handleReceived(). + * Verify shouldExhaustHops() returns false for any packet regardless of from/id. + * Since exhaust is removed, the from+id scope check is moot — this guards that + * the always-false invariant holds across multiple distinct packets. */ static void test_tm_alterReceived_exhaustFlag_isPacketScoped(void) { - moduleConfig.traffic_management.exhaust_hop_telemetry = true; - ScopedBusyAirTime busyChannel; // telemetry exhaust only fires under congestion TrafficManagementModuleTestShim module; - meshtastic_MeshPacket exhausted = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode, NODENUM_BROADCAST); - exhausted.id = 0x1010; - exhausted.hop_start = 5; - exhausted.hop_limit = 3; - module.alterReceived(exhausted); + meshtastic_MeshPacket p1 = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode, NODENUM_BROADCAST); + p1.id = 0x1010; + p1.hop_start = 5; + p1.hop_limit = 3; + module.alterReceived(p1); - meshtastic_MeshPacket unrelated = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kTargetNode, NODENUM_BROADCAST); - unrelated.id = 0x2020; - unrelated.hop_start = 4; - unrelated.hop_limit = 0; + meshtastic_MeshPacket p2 = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kTargetNode, NODENUM_BROADCAST); + p2.id = 0x2020; + p2.hop_start = 4; + p2.hop_limit = 0; - TEST_ASSERT_TRUE(module.shouldExhaustHops(exhausted)); - TEST_ASSERT_FALSE(module.shouldExhaustHops(unrelated)); + TEST_ASSERT_FALSE(module.shouldExhaustHops(p1)); + TEST_ASSERT_FALSE(module.shouldExhaustHops(p2)); } /** - * Verify runOnce() returns sleep-forever interval when module is disabled. - * Important to ensure the maintenance thread is effectively inert when off. + * Verify runOnce() returns sleep-forever interval when has_traffic_management is false. + * TMM has no runtime enable flag — the presence bit is the only runtime gate. */ static void test_tm_runOnce_disabledReturnsMaxInterval(void) { - moduleConfig.traffic_management.enabled = false; + moduleConfig.has_traffic_management = false; TrafficManagementModuleTestShim module; int32_t interval = module.runOnce(); @@ -1218,6 +1225,381 @@ static void test_tm_nextHop_keptAliveAcrossMaintenanceSweep(void) TEST_ASSERT_EQUAL_UINT8(0x42, module.getNextHopHint(kTargetNode)); } + +// --------------------------------------------------------------------------- +// Role exceptions: TRACKER and LOST_AND_FOUND +// --------------------------------------------------------------------------- + +/** + * Verify TRACKER role caps the dedup window at 1 hour. + * A duplicate position that would normally be blocked for 11 h (default) must + * be forwarded once the 1-hour tracker cap expires. + */ +static void test_tm_trackerRole_capsDedupWindowAtOneHour(void) +{ + // Operator interval is 11 h — longer than the tracker cap. + moduleConfig.traffic_management.position_min_interval_secs = default_traffic_mgmt_position_min_interval_secs; + installWellKnownPrimaryChannelWithPrecision(16); + + mockNodeDB->setCachedNode(kRemoteNode); + mockNodeDB->setCachedNodeRole(meshtastic_Config_DeviceConfig_Role_TRACKER); + + TrafficManagementModuleTestShim module; + + meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); + meshtastic_MeshPacket dup = makePositionPacket(kRemoteNode, 374221234, -1220845678); + meshtastic_MeshPacket afterCap = makePositionPacket(kRemoteNode, 374221234, -1220845678); + + ProcessMessage r1 = module.handleReceived(first); + ProcessMessage r2 = module.handleReceived(dup); // still within 1-hour cap + // Advance past 1 hour (tracker cap = 3600 s; pos-tick = 360 s → 10 ticks). + TrafficManagementModule::s_testNowMs += (default_traffic_mgmt_tracker_position_min_interval_secs * 1000UL) + 1; + ProcessMessage r3 = module.handleReceived(afterCap); + meshtastic_TrafficManagementStats stats = module.getStats(); + + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(r1)); + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::STOP), static_cast(r2)); + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(r3)); + TEST_ASSERT_EQUAL_UINT32(1, stats.position_dedup_drops); +} + +/** + * Verify TAK_TRACKER role receives the same 1-hour dedup cap as TRACKER. + * Both roles share the same exception branch; this guards against future divergence. + */ +static void test_tm_takTrackerRole_capsDedupWindowAtOneHour(void) +{ + moduleConfig.traffic_management.position_min_interval_secs = default_traffic_mgmt_position_min_interval_secs; + installWellKnownPrimaryChannelWithPrecision(16); + + mockNodeDB->setCachedNode(kRemoteNode); + mockNodeDB->setCachedNodeRole(meshtastic_Config_DeviceConfig_Role_TAK_TRACKER); + + TrafficManagementModuleTestShim module; + + meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); + meshtastic_MeshPacket dup = makePositionPacket(kRemoteNode, 374221234, -1220845678); + meshtastic_MeshPacket afterCap = makePositionPacket(kRemoteNode, 374221234, -1220845678); + + ProcessMessage r1 = module.handleReceived(first); + ProcessMessage r2 = module.handleReceived(dup); + TrafficManagementModule::s_testNowMs += (default_traffic_mgmt_tracker_position_min_interval_secs * 1000UL) + 1; + ProcessMessage r3 = module.handleReceived(afterCap); + meshtastic_TrafficManagementStats stats = module.getStats(); + + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(r1)); + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::STOP), static_cast(r2)); + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(r3)); + TEST_ASSERT_EQUAL_UINT32(1, stats.position_dedup_drops); +} + +/** + * Verify the tracker role exception survives the node being evicted from BOTH the + * hot and warm NodeDB stores — the TMM unified cache is the third fallback. The + * role is cached on the entry while NodeDB still knows the node; once NodeDB + * forgets it (getNodeRole → CLIENT), the cached role must keep the 1-hour cap + * applied instead of reverting to the 11-hour default interval. + */ +static void test_tm_trackerRole_survivesNodeDbEvictionViaCachedRole(void) +{ + // Operator interval is 11 h — longer than the tracker cap. + moduleConfig.traffic_management.position_min_interval_secs = default_traffic_mgmt_position_min_interval_secs; + installWellKnownPrimaryChannelWithPrecision(16); + + mockNodeDB->setCachedNode(kRemoteNode); + mockNodeDB->setCachedNodeRole(meshtastic_Config_DeviceConfig_Role_TRACKER); + + TrafficManagementModuleTestShim module; + + meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); + meshtastic_MeshPacket dup = makePositionPacket(kRemoteNode, 374221234, -1220845678); + meshtastic_MeshPacket afterCap = makePositionPacket(kRemoteNode, 374221234, -1220845678); + + // First packet: NodeDB knows the role; it is cached onto the TMM entry. + ProcessMessage r1 = module.handleReceived(first); + + // Node ages out of NodeDB entirely (hot + warm). getNodeRole now returns CLIENT. + mockNodeDB->clearCachedNode(); + + ProcessMessage r2 = module.handleReceived(dup); // within 1-hour cap — still drop + // Advance past the tracker cap (3600 s) but stay well under the 11-hour default. + // Without the cached-role fallback this would still be inside the 11-hour window + // (CLIENT → no exception) and wrongly drop; with it, the 1-hour cap lets it pass. + TrafficManagementModule::s_testNowMs += (default_traffic_mgmt_tracker_position_min_interval_secs * 1000UL) + 1; + ProcessMessage r3 = module.handleReceived(afterCap); + meshtastic_TrafficManagementStats stats = module.getStats(); + + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(r1)); + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::STOP), static_cast(r2)); + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(r3)); + TEST_ASSERT_EQUAL_UINT32(1, stats.position_dedup_drops); +} + +/** + * Verify a role change observed via NodeInfo updates the cached role (write-time update), + * so an exception is dropped when a node is demoted from TRACKER back to CLIENT. The role + * is read from the NodeInfo's User payload — the same event that updates NodeDB — not by + * re-scanning NodeDB on the position path. + */ +static void test_tm_roleChange_viaNodeInfo_dropsTrackerException(void) +{ + moduleConfig.traffic_management.position_min_interval_secs = default_traffic_mgmt_position_min_interval_secs; + installWellKnownPrimaryChannelWithPrecision(16); + + mockNodeDB->setCachedNode(kRemoteNode); + mockNodeDB->setCachedNodeRole(meshtastic_Config_DeviceConfig_Role_TRACKER); + + TrafficManagementModuleTestShim module; + + // First position seeds the TMM entry with TRACKER (from NodeDB on isNew). + meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); + ProcessMessage r1 = module.handleReceived(first); + + // The node demotes to CLIENT and announces it. The NodeInfo refresh must overwrite + // the cached TRACKER role with CLIENT — even though the packet payload, not NodeDB, + // is the source of truth here (NodeDB role left stale on purpose to prove the path). + meshtastic_MeshPacket info = makeNodeInfoPacketWithRole(kRemoteNode, meshtastic_Config_DeviceConfig_Role_CLIENT); + module.handleReceived(info); + + // Past the 1-hour tracker cap but within the 11-hour CLIENT interval. With the stale + // TRACKER role this would pass; after the demotion it must drop (full interval applies). + TrafficManagementModule::s_testNowMs += (default_traffic_mgmt_tracker_position_min_interval_secs * 1000UL) + 1; + meshtastic_MeshPacket afterCap = makePositionPacket(kRemoteNode, 374221234, -1220845678); + ProcessMessage r2 = module.handleReceived(afterCap); + meshtastic_TrafficManagementStats stats = module.getStats(); + + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(r1)); + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::STOP), static_cast(r2)); + TEST_ASSERT_EQUAL_UINT32(1, stats.position_dedup_drops); +} + +/** + * Verify a cached special (non-CLIENT) role pins the TMM entry through the maintenance + * sweep: once the node's position state has expired and been cleared, the entry — and its + * role — must still survive (role has no TTL), the same way a confirmed next-hop hint does. + */ +static void test_tm_specialRole_pinsEntryThroughSweep(void) +{ + // Short position interval → short pos TTL so the sweep clears pos state quickly. + moduleConfig.traffic_management.position_min_interval_secs = 300; + installWellKnownPrimaryChannelWithPrecision(16); + + mockNodeDB->setCachedNode(kRemoteNode); + mockNodeDB->setCachedNodeRole(meshtastic_Config_DeviceConfig_Role_TRACKER); + + TrafficManagementModuleTestShim module; + + module.handleReceived(makePositionPacket(kRemoteNode, 374221234, -1220845678)); + TEST_ASSERT_EQUAL_INT(static_cast(meshtastic_Config_DeviceConfig_Role_TRACKER), module.peekCachedRole(kRemoteNode)); + + // Advance well past the position TTL, then sweep: pos state is cleared but the role + // (no TTL) must keep the entry alive. Without the role pin the entry would be reclaimed + // and peekCachedRole would return -1. + TrafficManagementModule::s_testNowMs += 60UL * 60UL * 1000UL; // 1 h + module.runOnce(); + + TEST_ASSERT_EQUAL_INT(static_cast(meshtastic_Config_DeviceConfig_Role_TRACKER), module.peekCachedRole(kRemoteNode)); +} + +/** + * Verify special-role entries are evicted last. A tracker that is the OLDEST entry must + * survive cache pressure that evicts many newer CLIENT entries, because a cached + * special role marks the entry "preferred" (like a next-hop hint) in victim selection. + */ +static void test_tm_specialRole_evictedLastUnderPressure(void) +{ + moduleConfig.traffic_management.position_min_interval_secs = 300; + installWellKnownPrimaryChannelWithPrecision(16); + + TrafficManagementModuleTestShim module; + + // Oldest entry: a tracker. Seed its role from NodeDB on first position. + const NodeNum tracker = 0xAA0000FF; + mockNodeDB->setCachedNode(tracker); + mockNodeDB->setCachedNodeRole(meshtastic_Config_DeviceConfig_Role_TRACKER); + module.handleReceived(makePositionPacket(tracker, 100, 200)); + TEST_ASSERT_EQUAL_INT(static_cast(meshtastic_Config_DeviceConfig_Role_TRACKER), module.peekCachedRole(tracker)); + + mockNodeDB->clearCachedNode(); // every subsequent filler resolves to CLIENT (unprotected) + + // Fill past capacity with newer CLIENT nodes, forcing many evictions. The tracker is + // the stalest entry but is "preferred", so an unprotected client must always be the + // victim instead — the tracker must never be evicted. + for (uint32_t i = 0; i < static_cast(TRAFFIC_MANAGEMENT_CACHE_SIZE) + 50; i++) { + const NodeNum filler = 0x01000000u + i; + module.handleReceived(makePositionPacket(filler, 300 + static_cast(i), 400 + static_cast(i))); + } + + TEST_ASSERT_EQUAL_INT(static_cast(meshtastic_Config_DeviceConfig_Role_TRACKER), module.peekCachedRole(tracker)); +} + +/** + * Verify the tracker cap never lengthens an operator-configured interval shorter + * than the cap default. The cap is a ceiling, not a floor. + */ +static void test_tm_trackerRole_doesNotLengthenShorterOperatorInterval(void) +{ + // Operator set 5-minute interval — shorter than the 1-hour tracker cap. + moduleConfig.traffic_management.position_min_interval_secs = 300; + installWellKnownPrimaryChannelWithPrecision(16); + + mockNodeDB->setCachedNode(kRemoteNode); + mockNodeDB->setCachedNodeRole(meshtastic_Config_DeviceConfig_Role_TRACKER); + + TrafficManagementModuleTestShim module; + + meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); + meshtastic_MeshPacket dup = makePositionPacket(kRemoteNode, 374221234, -1220845678); + meshtastic_MeshPacket afterShortInterval = makePositionPacket(kRemoteNode, 374221234, -1220845678); + + ProcessMessage r1 = module.handleReceived(first); + ProcessMessage r2 = module.handleReceived(dup); + // The 300 s operator interval rounds to 1 pos-tick (360 s) — dedup is tick-granular. + // Advance past one full tick to verify the window is 1 tick (not the 10-tick tracker cap). + TrafficManagementModule::s_testNowMs += 360'000UL + 1; + ProcessMessage r3 = module.handleReceived(afterShortInterval); + meshtastic_TrafficManagementStats stats = module.getStats(); + + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(r1)); + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::STOP), static_cast(r2)); + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(r3)); + TEST_ASSERT_EQUAL_UINT32(1, stats.position_dedup_drops); +} + +/** + * Verify LOST_AND_FOUND role caps duplicate-position dedup at ~15 min (2 pos-ticks), + * not the old one-tick fast-announce. A configured 11-hour interval is shortened to the + * 15-min cap; a duplicate one tick later still drops, but one past the 2-tick cap passes. + */ +static void test_tm_lostAndFoundRole_capsDedupAtFifteenMinutes(void) +{ + // Long interval that would normally suppress duplicates for 11 h. + moduleConfig.traffic_management.position_min_interval_secs = default_traffic_mgmt_position_min_interval_secs; + installWellKnownPrimaryChannelWithPrecision(16); + + mockNodeDB->setCachedNode(kRemoteNode); + mockNodeDB->setCachedNodeRole(meshtastic_Config_DeviceConfig_Role_LOST_AND_FOUND); + + TrafficManagementModuleTestShim module; + + meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); + meshtastic_MeshPacket dup = makePositionPacket(kRemoteNode, 374221234, -1220845678); + meshtastic_MeshPacket afterOneTick = makePositionPacket(kRemoteNode, 374221234, -1220845678); + meshtastic_MeshPacket afterCap = makePositionPacket(kRemoteNode, 374221234, -1220845678); + + ProcessMessage r1 = module.handleReceived(first); + ProcessMessage r2 = module.handleReceived(dup); // same tick — drop + // One pos-tick later: STILL within the 15-min (~2-tick) cap, unlike the old 1-tick exception. + // (360 s = kPosTimeTickMs, kept as a literal because the constant is private to the module.) + TrafficManagementModule::s_testNowMs += 360'000UL + 1; + ProcessMessage r3 = module.handleReceived(afterOneTick); // still drop + // Jump past the full cap (>= 2 ticks since the last packet): now it passes. + TrafficManagementModule::s_testNowMs += 2 * 360'000UL; + ProcessMessage r4 = module.handleReceived(afterCap); + meshtastic_TrafficManagementStats stats = module.getStats(); + + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(r1)); + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::STOP), static_cast(r2)); + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::STOP), static_cast(r3)); + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(r4)); + TEST_ASSERT_EQUAL_UINT32(2, stats.position_dedup_drops); +} + +/** + * Verify a node with unknown role (absent from NodeDB) is not granted any role + * exception: the full configured interval applies, so a duplicate inside that + * window is dropped exactly as for an ordinary CLIENT. + */ +static void test_tm_unknownRole_noDbEntry_appliesFullInterval(void) +{ + moduleConfig.traffic_management.position_min_interval_secs = 300; + installWellKnownPrimaryChannelWithPrecision(16); + + // No cached node — getMeshNode returns nullptr for kRemoteNode. + mockNodeDB->clearCachedNode(); + + TrafficManagementModuleTestShim module; + + meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); + meshtastic_MeshPacket dup = makePositionPacket(kRemoteNode, 374221234, -1220845678); + meshtastic_MeshPacket afterShort = makePositionPacket(kRemoteNode, 374221234, -1220845678); + + ProcessMessage r1 = module.handleReceived(first); + ProcessMessage r2 = module.handleReceived(dup); // within 300-s window — must drop + // The 300 s operator interval rounds to 1 pos-tick (360 s) — dedup is tick-granular. + // Advance past one full tick to confirm the packet passes without any tracker exception. + TrafficManagementModule::s_testNowMs += 360'000UL + 1; + ProcessMessage r3 = module.handleReceived(afterShort); + meshtastic_TrafficManagementStats stats = module.getStats(); + + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(r1)); + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::STOP), static_cast(r2)); + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(r3)); + TEST_ASSERT_EQUAL_UINT32(1, stats.position_dedup_drops); +} + +/** + * Verify a node present in NodeDB but without user info (HAS_USER bit clear) + * is not granted a role exception: it falls back to CLIENT and the full + * configured interval applies. + */ +static void test_tm_unknownRole_noUserBit_appliesFullInterval(void) +{ + moduleConfig.traffic_management.position_min_interval_secs = 300; + installWellKnownPrimaryChannelWithPrecision(16); + + // Node is in NodeDB but the HAS_USER bit is NOT set — role must be ignored. + mockNodeDB->setCachedNode(kRemoteNode); + // Clear the HAS_USER bit that setCachedNode sets, leaving role at CLIENT default. + mockNodeDB->cachedNodeForTest().bitfield &= ~NODEINFO_BITFIELD_HAS_USER_MASK; + mockNodeDB->cachedNodeForTest().role = meshtastic_Config_DeviceConfig_Role_TRACKER; + + TrafficManagementModuleTestShim module; + + meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); + meshtastic_MeshPacket dup = makePositionPacket(kRemoteNode, 374221234, -1220845678); + + ProcessMessage r1 = module.handleReceived(first); + ProcessMessage r2 = module.handleReceived(dup); // within 300-s window — must drop + meshtastic_TrafficManagementStats stats = module.getStats(); + + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(r1)); + TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::STOP), static_cast(r2)); + TEST_ASSERT_EQUAL_UINT32(1, stats.position_dedup_drops); +} + +/** + * Verify a LOST_AND_FOUND origin now GETS the relayed precision clamp — the + * anti-dox exemption was removed, so a relayed position more precise than the + * channel setting is clamped down to the channel ceiling like any other node's. + */ +static void test_tm_lostAndFoundRole_getsAlterReceivedPrecisionClamp(void) +{ + // Set channel precision ceiling to 13 bits. Must be <= MAX_POSITION_PRECISION_PUBLIC_KEY + // (15) — well-known channels have a public PSK (size==1), so getPositionPrecisionForChannel + // clamps any value above 15 via usesPublicKey(). + installWellKnownPrimaryChannelWithPrecision(13); + + mockNodeDB->setCachedNode(kRemoteNode); + mockNodeDB->setCachedNodeRole(meshtastic_Config_DeviceConfig_Role_LOST_AND_FOUND); + + TrafficManagementModuleTestShim module; + + // Full-precision packet — 32 bits — exceeds the channel cap. + const uint32_t fullPrecision = 32; + meshtastic_MeshPacket packet = makePositionPacketWithPrecision(kRemoteNode, 374221234, -1220845678, fullPrecision); + packet.hop_start = 3; + packet.hop_limit = 2; // relayed (hop_limit < hop_start) so clamp logic applies + + module.alterReceived(packet); + + meshtastic_Position out; + TEST_ASSERT_TRUE(decodePositionPayload(packet, out)); + // Clamped to channel ceiling (13 bits) — lost-and-found is no longer exempt. + // Note: precision must be <= MAX_POSITION_PRECISION_PUBLIC_KEY (15); well-known + // channels always have a public PSK so getPositionPrecisionForChannel caps at 15. + TEST_ASSERT_EQUAL_UINT32(13, out.precision_bits); +} } // namespace void setUp(void) @@ -1254,21 +1636,21 @@ TM_TEST_ENTRY void setup() RUN_TEST(test_tm_nodeinfo_directResponse_psramCacheRespondsAndPreservesBitfield); RUN_TEST(test_tm_nodeinfo_directResponse_psramMissDoesNotFallbackToNodeDb); #endif - RUN_TEST(test_tm_alterReceived_exhaustsRelayedTelemetryBroadcast); + RUN_TEST(test_tm_alterReceived_telemetryBroadcast_hopLimitUnchanged); RUN_TEST(test_tm_alterReceived_skipsLocalAndUnicast); RUN_TEST(test_tm_positionDedup_allowsDuplicateAfterIntervalExpires); RUN_TEST(test_tm_positionDedup_intervalZero_neverDrops); RUN_TEST(test_tm_positionDedup_precisionAbove32_usesDefaultPrecision); - RUN_TEST(test_tm_positionDedup_precision32_allowsDistinctPositions); - RUN_TEST(test_tm_positionDedup_precisionZero_allowsDistinctPositions); - RUN_TEST(test_tm_positionDedup_epochReset_doesNotDropFirstPacketAfterReset); + RUN_TEST(test_tm_positionDedup_distinctAtClampedChannelPrecision); + RUN_TEST(test_tm_positionDedup_precisionZero_channelFallsBackToDefault); + RUN_TEST(test_tm_positionDedup_cacheFlush_doesNotDropFirstPacketAfterFlush); RUN_TEST(test_tm_positionDedup_priorRateState_doesNotDropFirstFingerprintZero); RUN_TEST(test_tm_rateLimit_resetsAfterWindowExpires); RUN_TEST(test_tm_unknownPackets_resetAfterWindowExpires); RUN_TEST(test_tm_unknownPackets_thresholdAbove255_clamps); - RUN_TEST(test_tm_alterReceived_exhaustsRelayedPositionBroadcast); + RUN_TEST(test_tm_alterReceived_positionBroadcast_hopLimitUnchanged); RUN_TEST(test_tm_alterReceived_skipsUndecodedPackets); - RUN_TEST(test_tm_alterReceived_resetExhaustFlagOnNextPacket); + RUN_TEST(test_tm_alterReceived_exhaustFlagAlwaysFalse); RUN_TEST(test_tm_alterReceived_exhaustFlag_isPacketScoped); RUN_TEST(test_tm_runOnce_disabledReturnsMaxInterval); RUN_TEST(test_tm_runOnce_enabledReturnsMaintenanceInterval); @@ -1276,6 +1658,17 @@ TM_TEST_ENTRY void setup() RUN_TEST(test_tm_nextHop_servedAfterNodeDbRoll); RUN_TEST(test_tm_nextHop_preloadDoesNotClobberLearned); RUN_TEST(test_tm_nextHop_keptAliveAcrossMaintenanceSweep); + RUN_TEST(test_tm_trackerRole_capsDedupWindowAtOneHour); + RUN_TEST(test_tm_takTrackerRole_capsDedupWindowAtOneHour); + RUN_TEST(test_tm_trackerRole_survivesNodeDbEvictionViaCachedRole); + RUN_TEST(test_tm_roleChange_viaNodeInfo_dropsTrackerException); + RUN_TEST(test_tm_specialRole_pinsEntryThroughSweep); + RUN_TEST(test_tm_specialRole_evictedLastUnderPressure); + RUN_TEST(test_tm_trackerRole_doesNotLengthenShorterOperatorInterval); + RUN_TEST(test_tm_lostAndFoundRole_capsDedupAtFifteenMinutes); + RUN_TEST(test_tm_lostAndFoundRole_getsAlterReceivedPrecisionClamp); + RUN_TEST(test_tm_unknownRole_noDbEntry_appliesFullInterval); + RUN_TEST(test_tm_unknownRole_noUserBit_appliesFullInterval); exit(UNITY_END()); } diff --git a/userPrefs.jsonc b/userPrefs.jsonc index 57ede8bbf..118588e19 100644 --- a/userPrefs.jsonc +++ b/userPrefs.jsonc @@ -24,6 +24,7 @@ // "USERPREFS_CONFIG_OWNER_SHORT_NAME": "MLN", // "USERPREFS_CONFIG_DEVICE_ROLE": "meshtastic_Config_DeviceConfig_Role_CLIENT", // Defaults to CLIENT. ROUTER*, and LOST AND FOUND roles are restricted. // "USERPREFS_EVENT_MODE": "1", + // "USERPREFS_TMM_APPLY_TO_PRIVATE_CHANNELS": "1", // Extend TMM position dedup and precision clamping to private/custom-key channels (default: well-known channels only) // "USERPREFS_FIRMWARE_EDITION": "meshtastic_FirmwareEdition_BURNING_MAN", // "USERPREFS_FIXED_BLUETOOTH": "121212", // "USERPREFS_FIXED_GPS": "", diff --git a/variants/esp32s3/heltec_v4/variant.h b/variants/esp32s3/heltec_v4/variant.h index 5a806653e..d35cce554 100644 --- a/variants/esp32s3/heltec_v4/variant.h +++ b/variants/esp32s3/heltec_v4/variant.h @@ -33,9 +33,6 @@ #ifndef HAS_TRAFFIC_MANAGEMENT #define HAS_TRAFFIC_MANAGEMENT 1 #endif -#ifndef TRAFFIC_MANAGEMENT_CACHE_SIZE -#define TRAFFIC_MANAGEMENT_CACHE_SIZE 2048 -#endif // ---- GC1109 RF FRONT END CONFIGURATION ---- // The Heltec V4.2 uses a GC1109 FEM chip with integrated PA and LNA diff --git a/variants/esp32s3/heltec_v4_r8/variant.h b/variants/esp32s3/heltec_v4_r8/variant.h index d59f0ae2c..b7d8897a5 100644 --- a/variants/esp32s3/heltec_v4_r8/variant.h +++ b/variants/esp32s3/heltec_v4_r8/variant.h @@ -31,9 +31,6 @@ #ifndef HAS_TRAFFIC_MANAGEMENT #define HAS_TRAFFIC_MANAGEMENT 1 #endif -#ifndef TRAFFIC_MANAGEMENT_CACHE_SIZE -#define TRAFFIC_MANAGEMENT_CACHE_SIZE 2048 -#endif // ---- KCT8103L RF FRONT END CONFIGURATION ---- // The Heltec V4.3 uses a KCT8103L FEM chip with integrated PA and LNA diff --git a/variants/esp32s3/station-g2/variant.h b/variants/esp32s3/station-g2/variant.h index 9e844588f..fdd7c280d 100755 --- a/variants/esp32s3/station-g2/variant.h +++ b/variants/esp32s3/station-g2/variant.h @@ -14,9 +14,6 @@ Board Information: https://wiki.uniteng.com/en/meshtastic/station-g2 #ifndef HAS_TRAFFIC_MANAGEMENT #define HAS_TRAFFIC_MANAGEMENT 1 #endif -#ifndef TRAFFIC_MANAGEMENT_CACHE_SIZE -#define TRAFFIC_MANAGEMENT_CACHE_SIZE 2048 -#endif /* #define BATTERY_PIN 4 // A battery voltage measurement pin, voltage divider connected here to measure battery voltage diff --git a/variants/native/portduino/variant.h b/variants/native/portduino/variant.h index fb36e2588..7ccf2e34b 100644 --- a/variants/native/portduino/variant.h +++ b/variants/native/portduino/variant.h @@ -16,6 +16,3 @@ #ifndef HAS_VARIABLE_HOPS #define HAS_VARIABLE_HOPS 1 #endif -#ifndef TRAFFIC_MANAGEMENT_CACHE_SIZE -#define TRAFFIC_MANAGEMENT_CACHE_SIZE 2048 -#endif From 3501f620a30adb245b9bc00ff4b99b7c39f1bfcc Mon Sep 17 00:00:00 2001 From: Carlos Valdes Date: Sun, 21 Jun 2026 15:17:09 +0200 Subject: [PATCH 23/86] fix: restore fixed position into localPosition on boot (#10670) * NodeDB: restore fixed position into localPosition at boot Fixed-position nodes without a GPS stop broadcasting their position (and publishing MQTT map reports) after a reboot. On boot localPosition is empty, and NodeDB::hasValidPosition() for the local node only inspects localPosition, not the persisted node position. PositionModule therefore never sends a position, so the lazy localPosition backfill in getPositionPacket() never runs and localPosition stays at (0,0) indefinitely. Restore the configured fixed position into localPosition during NodeDB construction so position broadcasts and map reports resume automatically after a reboot. * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Ben Meadors Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/mesh/NodeDB.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index 27c498e97..802621f70 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -590,6 +590,16 @@ NodeDB::NodeDB() moduleConfig.mqtt.map_report_settings.publish_interval_secs = default_map_publish_interval_secs; } + // If a fixed position is configured, restore the persisted position into localPosition at boot. + // This keeps position broadcasts / MQTT map reports working after reboot on GPS-less nodes. + if (config.position.fixed_position) { + meshtastic_PositionLite fixedPos; + if (copyNodePosition(getNodeNum(), fixedPos) && (fixedPos.latitude_i != 0 || fixedPos.longitude_i != 0)) { + setLocalPosition(TypeConversions::ConvertToPosition(fixedPos)); + LOG_INFO("Restored fixed position to localPosition: lat=%d lon=%d", fixedPos.latitude_i, fixedPos.longitude_i); + } + } + // Ensure that the neighbor info update interval is coerced to the minimum moduleConfig.neighbor_info.update_interval = Default::getConfiguredOrMinimumValue(moduleConfig.neighbor_info.update_interval, min_neighbor_info_broadcast_secs); From c4fdf374e144c7de6bb5b2692ee127b5de74d86f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=B6ttgens?= Date: Mon, 22 Jun 2026 13:21:50 +0200 Subject: [PATCH 24/86] fix: right-size warm tier for constrained platforms (#10746, #10705) (#10759) * fix: right-size warm tier for constrained platforms and feed RP2040 watchdog during NodeDB save * fix: size warm tier and traffic cache per-MCU RAM, lowering no-PSRAM ESP32 (classic/S2/C3) tiers * docs: document nRF52/RP2040 #else fall-through in warm tier and TM cache cascades --- src/mesh/NodeDB.cpp | 9 +++++++++ src/mesh/mesh-pb-constants.h | 22 +++++++++++++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index 802621f70..05d36c8af 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -66,6 +66,10 @@ #include #endif +#ifdef ARCH_RP2040 +#include +#endif + #if defined(ARCH_ESP32) && !MESHTASTIC_EXCLUDE_WIFI #include #endif @@ -2682,6 +2686,11 @@ bool NodeDB::saveNodeDatabaseToDisk() nodeDatabase.status.clear(); nodeDatabase.status.shrink_to_fit(); #if WARM_NODE_COUNT > 0 +#ifdef ARCH_RP2040 + // nodes.proto + warm.dat are written back-to-back without the loop running between them; + // reset the 8s HW watchdog so the second write gets a full budget (issue #10746). + watchdog_update(); +#endif // Same cadence as the node DB; failure is logged but must not propagate — // a false return from here would trigger saveToDisk()'s fsFormat() path. warmStore.saveIfDirty(); diff --git a/src/mesh/mesh-pb-constants.h b/src/mesh/mesh-pb-constants.h index 375be3ae5..7d64bc058 100644 --- a/src/mesh/mesh-pb-constants.h +++ b/src/mesh/mesh-pb-constants.h @@ -128,9 +128,18 @@ static inline int get_max_num_nodes() // chain. Backed by the raw-flash ring below LittleFS — see WarmNodeStore.h. #define WARM_NODE_COUNT 200 #elif (defined(CONFIG_IDF_TARGET_ESP32S3) && defined(BOARD_HAS_PSRAM)) || defined(ARCH_PORTDUINO) -#define WARM_NODE_COUNT 2000 // PSRAM-equipped ESP32-S3 / native host; warm.dat ~80 KB +#define WARM_NODE_COUNT 2000 // PSRAM-equipped ESP32-S3 / native host; warm cache in PSRAM (~80 KB) +#elif defined(CONFIG_IDF_TARGET_ESP32S3) || defined(CONFIG_IDF_TARGET_ESP32C6) || defined(CONFIG_IDF_TARGET_ESP32P4) +#define WARM_NODE_COUNT 150 // 512 KB+ SRAM, no PSRAM (S3/C6/P4): ~6 KB heap (#10705) +#elif defined(ARCH_ESP32) +#define WARM_NODE_COUNT 100 // classic ESP32 (520 KB) / S2 (320 KB) / C3 (400 KB): tightest free heap w/ BLE+WiFi, ~4 KB (#10705) +#elif defined(ARCH_RP2040) +#define WARM_NODE_COUNT 150 // RP2040 (264 KB) / RP2350 (520 KB): bounded so warm.dat write fits the 8s watchdog (#10746) #else -#define WARM_NODE_COUNT 320 // Generic ESP32, ESP32-S3 without PSRAM, ESP32C3 etc. +// nRF52840 is handled explicitly above (200, raw-flash ring). Any other nRF52 (non-XXAA) and any +// future non-ESP32/non-RP LittleFS part fall through to this 320 default — flag for review if such a +// RAM-constrained nRF52 target is ever added. +#define WARM_NODE_COUNT 320 // other LittleFS-backed parts (e.g. non-nRF52840 nRF52) #endif // platform #endif // WARM_NODE_COUNT @@ -164,8 +173,15 @@ static inline int get_max_num_nodes() #define TRAFFIC_MANAGEMENT_CACHE_SIZE 0 #elif (defined(CONFIG_IDF_TARGET_ESP32S3) && defined(BOARD_HAS_PSRAM)) || defined(ARCH_PORTDUINO) #define TRAFFIC_MANAGEMENT_CACHE_SIZE 2048 // PSRAM-equipped ESP32-S3 / native host +#elif defined(CONFIG_IDF_TARGET_ESP32S3) || defined(CONFIG_IDF_TARGET_ESP32C6) || defined(CONFIG_IDF_TARGET_ESP32P4) +#define TRAFFIC_MANAGEMENT_CACHE_SIZE 500 // 512 KB+ SRAM, no PSRAM (S3/C6/P4): ~5 KB heap (#10705) +#elif defined(ARCH_ESP32) +#define TRAFFIC_MANAGEMENT_CACHE_SIZE 400 // classic ESP32 / S2 / C3: tightest free heap, ~4 KB (#10705) #else -#define TRAFFIC_MANAGEMENT_CACHE_SIZE 1000 // Generic ESP32, ESP32-S3 without PSRAM +// nRF52 (incl. nRF52840) and RP2040/RP2350 fall through here — there is no nRF/RP branch above, +// by design. These parts have no ESP32-style WiFi+BLE coexistence eating the heap, so the larger +// 1000-entry (~10 KB) cache fits: nRF52840 is BLE-only on 256 KB RAM; RP2040/RP2350 have 264/520 KB. +#define TRAFFIC_MANAGEMENT_CACHE_SIZE 1000 // nRF52 / RP2040 / RP2350 / other non-ESP32 #endif #endif // TRAFFIC_MANAGEMENT_CACHE_SIZE From 04545c7ba2f166b76e4d2b985057dca3f408643b Mon Sep 17 00:00:00 2001 From: Jonathan Bennett Date: Mon, 22 Jun 2026 16:31:49 -0500 Subject: [PATCH 25/86] Delete dead code src/platform/esp32/iram-quirk.c (#10762) --- src/platform/esp32/iram-quirk.c | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 src/platform/esp32/iram-quirk.c diff --git a/src/platform/esp32/iram-quirk.c b/src/platform/esp32/iram-quirk.c deleted file mode 100644 index 813842138..000000000 --- a/src/platform/esp32/iram-quirk.c +++ /dev/null @@ -1,23 +0,0 @@ -// Free up some precious space in the iram0_0_seg memory segment - -#include - -#include -#include -#include - -#define IRAM_SECTION section(".iram1.stub") - -IRAM_ATTR esp_err_t stub_probe(esp_flash_t *chip, uint32_t flash_id) -{ - return ESP_ERR_NOT_FOUND; -} - -const spi_flash_chip_t stub_flash_chip __attribute__((IRAM_SECTION)) = { - .name = "stub", - .probe = stub_probe, -}; - -extern const spi_flash_chip_t __wrap_esp_flash_chip_gd __attribute__((IRAM_SECTION, alias("stub_flash_chip"))); -extern const spi_flash_chip_t __wrap_esp_flash_chip_issi __attribute__((IRAM_SECTION, alias("stub_flash_chip"))); -extern const spi_flash_chip_t __wrap_esp_flash_chip_winbond __attribute__((IRAM_SECTION, alias("stub_flash_chip"))); From fcfe329091f2c7861dde0d6dd2fce1aa29057ba1 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Tue, 23 Jun 2026 05:31:17 -0500 Subject: [PATCH 26/86] Guard TCP API writes after Wi-Fi reconnects (#10505) * Guard TCP API writes against dead sockets Agent-Logs-Url: https://github.com/meshtastic/firmware/sessions/ebeb38d5-7339-4eac-b310-4b6dd9d40758 Co-authored-by: thebentern <9000580+thebentern@users.noreply.github.com> * fix: apply review fixes for server write checks and stream locking Agent-Logs-Url: https://github.com/meshtastic/firmware/sessions/215c659d-bc17-4b30-89f4-78e3f9cdb3c3 Co-authored-by: thebentern <9000580+thebentern@users.noreply.github.com> * style: format TCP API write check files * fix(api): keep TCP API open on write backpressure; drop broken writability gate canWriteFrame() treated a full transmit buffer (availableForWrite() == 0) as a dead socket and closed the connection, so every healthy client was dropped and the native simulator integration test timed out waiting for config. A zero availableForWrite() is normal backpressure, not a disconnect; only a dropped link should refuse a write. Reduce canWriteFrame() to the reliable !client.connected() check -- genuine write failures are still caught after the fact by onFrameWriteFailed(). Remove the now-unused ClientWriteChecks.h writability helper and its unit test (test_client_write_checks, which also failed to link), and restore ServerAPI.cpp to the project clang-format style (fixes the Trunk check). Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: thebentern <9000580+thebentern@users.noreply.github.com> Co-authored-by: Ben Meadors Co-authored-by: Claude Opus 4.8 --- src/mesh/StreamAPI.cpp | 57 ++++++++++++++++++++------------------ src/mesh/StreamAPI.h | 9 ++++-- src/mesh/api/ServerAPI.cpp | 30 ++++++++++++++++++-- src/mesh/api/ServerAPI.h | 2 ++ 4 files changed, 67 insertions(+), 31 deletions(-) diff --git a/src/mesh/StreamAPI.cpp b/src/mesh/StreamAPI.cpp index 2c3362877..9785b7f5e 100644 --- a/src/mesh/StreamAPI.cpp +++ b/src/mesh/StreamAPI.cpp @@ -52,7 +52,8 @@ void StreamAPI::writeStream() do { // Send every packet we can len = getFromRadio(txBuf + HEADER_LEN); - emitTxBuffer(len); + if (len != 0 && !emitTxBuffer(len)) + break; } while (len); } } @@ -169,21 +170,36 @@ int32_t StreamAPI::readStream() /** * Send the current txBuffer over our stream */ -void StreamAPI::emitTxBuffer(size_t len) +bool StreamAPI::writeFrame(uint8_t *buf, size_t len) { - if (len != 0) { - txBuf[0] = START1; - txBuf[1] = START2; - txBuf[2] = (len >> 8) & 0xff; - txBuf[3] = len & 0xff; + if (len == 0 || !canWrite) + return false; - auto totalLen = len + HEADER_LEN; - // Serialize stream writes against `emitLogRecord` so a LOG_ firing - // mid-packet-emission can't interleave bytes on the wire. - concurrency::LockGuard guard(&streamLock); - stream->write(txBuf, totalLen); + buf[0] = START1; + buf[1] = START2; + buf[2] = (len >> 8) & 0xff; + buf[3] = len & 0xff; + + auto totalLen = len + HEADER_LEN; + // Serialize write-readiness checks, writes and write-failure handling + // against concurrent stream writes/close. + concurrency::LockGuard guard(&streamLock); + if (!canWriteFrame(totalLen)) + return false; + + size_t written = stream->write(buf, totalLen); + if (written == totalLen) { stream->flush(); + return true; } + + onFrameWriteFailed(totalLen, written); + return false; +} + +bool StreamAPI::emitTxBuffer(size_t len) +{ + return writeFrame(txBuf, len); } void StreamAPI::emitRebooted() @@ -221,20 +237,7 @@ void StreamAPI::emitLogRecord(meshtastic_LogRecord_Level level, const char *src, size_t len = pb_encode_to_bytes(txBufLog + HEADER_LEN, meshtastic_FromRadio_size, &meshtastic_FromRadio_msg, &fromRadioScratchLog); - if (len != 0) { - txBufLog[0] = START1; - txBufLog[1] = START2; - txBufLog[2] = (len >> 8) & 0xff; - txBufLog[3] = len & 0xff; - - auto totalLen = len + HEADER_LEN; - // Serialize stream writes against `emitTxBuffer` so a packet - // emission in flight on another task doesn't interleave bytes - // with this log record. - concurrency::LockGuard guard(&streamLock); - stream->write(txBufLog, totalLen); - stream->flush(); - } + writeFrame(txBufLog, len); } /// Hookable to find out when connection changes @@ -249,4 +252,4 @@ void StreamAPI::onConnectionChanged(bool connected) // received a packet in a while powerFSM.trigger(EVENT_SERIAL_DISCONNECTED); } -} \ No newline at end of file +} diff --git a/src/mesh/StreamAPI.h b/src/mesh/StreamAPI.h index d3ad9ba79..65a2758af 100644 --- a/src/mesh/StreamAPI.h +++ b/src/mesh/StreamAPI.h @@ -80,7 +80,7 @@ class StreamAPI : public PhoneAPI /** * Send the current txBuffer over our stream */ - void emitTxBuffer(size_t len); + bool emitTxBuffer(size_t len); /// Are we allowed to write packets to our output stream (subclasses can turn this off - i.e. SerialConsole) bool canWrite = true; @@ -91,7 +91,12 @@ class StreamAPI : public PhoneAPI /// Low level function to emit a protobuf encapsulated log record void emitLogRecord(meshtastic_LogRecord_Level level, const char *src, const char *format, va_list arg); + virtual bool canWriteFrame(size_t frameLen) { return true; } + virtual void onFrameWriteFailed(size_t frameLen, size_t writtenLen) {} + private: + bool writeFrame(uint8_t *buf, size_t len); + /// Dedicated scratch + tx buffer for LogRecord emission. /// /// The main packet emission path (`writeStream` -> `getFromRadio` -> @@ -113,4 +118,4 @@ class StreamAPI : public PhoneAPI meshtastic_FromRadio fromRadioScratchLog = {}; uint8_t txBufLog[MAX_STREAM_BUF_SIZE] = {0}; concurrency::Lock streamLock; -}; \ No newline at end of file +}; diff --git a/src/mesh/api/ServerAPI.cpp b/src/mesh/api/ServerAPI.cpp index f3e7854ca..af6790cd2 100644 --- a/src/mesh/api/ServerAPI.cpp +++ b/src/mesh/api/ServerAPI.cpp @@ -22,12 +22,38 @@ template void ServerAPI::close() StreamAPI::close(); } -/// Check the current underlying physical link to see if the client is currently connected +/// Check the current underlying physical link to see if the client is currently +/// connected template bool ServerAPI::checkIsConnected() { return client.connected(); } +template bool ServerAPI::canWriteFrame(size_t) +{ + // Only a dropped link is a reason to refuse a write up front. A full transmit + // buffer (availableForWrite() == 0) is normal backpressure, not a dead socket, + // so we must not close the connection on it. A genuinely failed write is + // detected after the fact in onFrameWriteFailed(). + if (!client.connected()) { + canWrite = false; + enabled = false; + LOG_WARN("TCP client disconnected before write, closing API service"); + close(); + return false; + } + + return true; +} + +template void ServerAPI::onFrameWriteFailed(size_t frameLen, size_t writtenLen) +{ + canWrite = false; + enabled = false; + LOG_WARN("TCP client write short (%lu/%lu bytes), closing API service", (unsigned long)writtenLen, (unsigned long)frameLen); + close(); +} + template int32_t ServerAPI::runOnce() { if (client.connected()) { @@ -95,4 +121,4 @@ template int32_t APIServerPort::runOnce() waitTime = 100; #endif return 100; // only check occasionally for incoming connections -} \ No newline at end of file +} diff --git a/src/mesh/api/ServerAPI.h b/src/mesh/api/ServerAPI.h index 2da77c8e9..ece8e0ba2 100644 --- a/src/mesh/api/ServerAPI.h +++ b/src/mesh/api/ServerAPI.h @@ -29,6 +29,8 @@ template class ServerAPI : public StreamAPI, private concurrency::OSTh /// We override this method to prevent publishing EVENT_SERIAL_CONNECTED/DISCONNECTED for wifi links (we want the board to /// stay in the POWERED state to prevent disabling wifi) virtual void onConnectionChanged(bool connected) override {} + virtual bool canWriteFrame(size_t frameLen) override; + virtual void onFrameWriteFailed(size_t frameLen, size_t writtenLen) override; virtual int32_t runOnce() override; // Check for dropped client connections }; From ec82aa116d05ca1be6f24455a72468585352d5f6 Mon Sep 17 00:00:00 2001 From: jessm33 Date: Wed, 24 Jun 2026 09:24:41 -0400 Subject: [PATCH 27/86] Update native to exit when configured with an unknown module instead of going in to sim mode. --- src/platform/portduino/PortduinoGlue.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/platform/portduino/PortduinoGlue.cpp b/src/platform/portduino/PortduinoGlue.cpp index c3cb1c780..2c72d83cb 100644 --- a/src/platform/portduino/PortduinoGlue.cpp +++ b/src/platform/portduino/PortduinoGlue.cpp @@ -784,12 +784,19 @@ bool loadConfig(const char *configPath) if (yamlConfig["Lora"]) { if (yamlConfig["Lora"]["Module"]) { + const std::string moduleName = yamlConfig["Lora"]["Module"].as(""); + bool found = false; for (const auto &loraModule : portduino_config.loraModules) { - if (yamlConfig["Lora"]["Module"].as("") == loraModule.second) { + if (moduleName == loraModule.second) { portduino_config.lora_module = loraModule.first; + found = true; break; } } + if (!found) { + std::cerr << "Unknown Lora.Module: " << moduleName << std::endl; + exit(EXIT_FAILURE); + } } if (yamlConfig["Lora"]["SX126X_MAX_POWER"]) portduino_config.sx126x_max_power = yamlConfig["Lora"]["SX126X_MAX_POWER"].as(22); From b93239bb0a7a2b642fab90de2d5e62c7195a8ec5 Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Wed, 24 Jun 2026 12:38:46 -0500 Subject: [PATCH 28/86] fix(phone-api): skip manifest scan for node-info-only requests; bound getFiles (#10778) Forward-port of #10754 and #10757 from master (2.7) into develop, so the develop->master 2.8 promotion (#10777) doesn't drop them. #10754: PhoneAPI no longer walks the filesystem to build the file manifest on node-info-only config requests (SPECIAL_NONCE_ONLY_NODES), which never consume it. getFiles() is now bounded (default 64 entries, depth 3) via collectFiles(), takes an optional wasLimited out-param, and reserves capacity with a bad_alloc/ length_error fallback. The manifest vector is freed via swap (releaseFilesManifest). #10757: getFiles()/collectFiles() now guard against empty file names returned by the Adafruit LittleFS nRF52 glue (issue 4395). Ported by hand rather than cherry-picked: master had reflowed FSCommon.cpp to a different brace style (every line conflicted), #10754 already subsumes #10757, and develop carries a MESHTASTIC_EXCLUDE_FILES_MANIFEST path (nRF54L15) that master lacks. The exclude path is preserved and now also short-circuits + frees the manifest. Verified: native Docker suite 448/448, clang-format clean. --- src/FSCommon.cpp | 149 +++++++++++++++++++++++++++++++++--------- src/FSCommon.h | 2 +- src/mesh/PhoneAPI.cpp | 39 ++++++++--- 3 files changed, 150 insertions(+), 40 deletions(-) diff --git a/src/FSCommon.cpp b/src/FSCommon.cpp index a229dcbfc..7b0595aa9 100644 --- a/src/FSCommon.cpp +++ b/src/FSCommon.cpp @@ -88,6 +88,9 @@ bool renameFile(const char *pathFrom, const char *pathTo) #endif } +#include +#include +#include #include /** @@ -119,6 +122,93 @@ bool fsFormat() #endif } +#ifdef FSCom +namespace +{ +bool pathEndsWithDot(const char *path) +{ + if (!path) + return false; + + size_t length = strlen(path); + return length > 0 && path[length - 1] == '.'; +} + +bool copyFilePath(char *dest, size_t destSize, const char *path, bool *wasLimited) +{ + if (!path || destSize == 0) { + if (wasLimited) + *wasLimited = true; + return false; + } + + if (strlcpy(dest, path, destSize) >= destSize) { + if (wasLimited) + *wasLimited = true; + return false; + } + + return true; +} + +void collectFiles(const char *dirname, uint8_t levels, size_t maxCount, std::vector &filenames, + bool *wasLimited) +{ + if (!dirname) + return; + + File root = FSCom.open(dirname, FILE_O_READ); + if (!root) + return; + if (!root.isDirectory()) { + root.close(); + return; + } + + File file = root.openNextFile(); + // file.name()[0] check is a workaround for a bug in the Adafruit LittleFS nrf52 glue (see issue 4395) + while (file && file.name()[0]) { + if (filenames.size() >= maxCount) { + if (wasLimited) + *wasLimited = true; + file.close(); + break; + } + const char *fileName = file.name(); + if (file.isDirectory() && !pathEndsWithDot(fileName)) { + char pathBuffer[sizeof(((meshtastic_FileInfo *)nullptr)->file_name)] = {}; +#ifdef ARCH_ESP32 + const char *subDirPath = file.path(); +#else + const char *subDirPath = fileName; +#endif + bool hasSubDirPath = copyFilePath(pathBuffer, sizeof(pathBuffer), subDirPath, wasLimited); + file.close(); + + if (levels && hasSubDirPath) { + collectFiles(pathBuffer, levels - 1, maxCount, filenames, wasLimited); + } else if (wasLimited) { + *wasLimited = true; + } + } else { + meshtastic_FileInfo fileInfo = {"", static_cast(file.size())}; +#ifdef ARCH_ESP32 + bool hasFilePath = copyFilePath(fileInfo.file_name, sizeof(fileInfo.file_name), file.path(), wasLimited); +#else + bool hasFilePath = copyFilePath(fileInfo.file_name, sizeof(fileInfo.file_name), file.name(), wasLimited); +#endif + if (hasFilePath && !pathEndsWithDot(fileInfo.file_name)) { + filenames.push_back(fileInfo); + } + file.close(); + } + file = root.openNextFile(); + } + root.close(); +} +} // namespace +#endif + /** * @brief Get the list of files in a directory. * @@ -127,43 +217,40 @@ bool fsFormat() * * @param dirname The name of the directory. * @param levels The number of levels of subdirectories to list. - * @return A vector of strings containing the full path of each file in the directory. + * @param maxCount The maximum number of files to collect before truncating the walk. + * @param wasLimited Optional out-param, set to true if the listing was truncated (by maxCount or low memory). + * @return A vector of meshtastic_FileInfo for each file in the directory. */ -std::vector getFiles(const char *dirname, uint8_t levels) +std::vector getFiles(const char *dirname, uint8_t levels, size_t maxCount, bool *wasLimited) { std::vector filenames = {}; + if (wasLimited) + *wasLimited = false; #ifdef FSCom - File root = FSCom.open(dirname, FILE_O_READ); - if (!root) - return filenames; - if (!root.isDirectory()) - return filenames; - - File file = root.openNextFile(); - while (file) { -#ifdef ARCH_ESP32 - const char *filepath = file.path(); -#else - const char *filepath = file.name(); -#endif - if (file.isDirectory() && !String(file.name()).endsWith(".")) { - if (levels) { - std::vector subDirFilenames = getFiles(filepath, levels - 1); - filenames.insert(filenames.end(), subDirFilenames.begin(), subDirFilenames.end()); - file.close(); - } - } else { - meshtastic_FileInfo fileInfo = {"", static_cast(file.size())}; - strncpy(fileInfo.file_name, filepath, sizeof(fileInfo.file_name) - 1); - fileInfo.file_name[sizeof(fileInfo.file_name) - 1] = '\0'; - if (!String(fileInfo.file_name).endsWith(".")) { - filenames.push_back(fileInfo); - } - file.close(); +#if defined(__cpp_exceptions) || defined(__EXCEPTIONS) + size_t reservedCount = maxCount; + while (reservedCount > 0) { + try { + filenames.reserve(reservedCount); + break; + } catch (const std::bad_alloc &) { + reservedCount /= 2; + } catch (const std::length_error &) { + reservedCount /= 2; } - file = root.openNextFile(); } - root.close(); + if (reservedCount == 0) { + if (wasLimited) + *wasLimited = true; + return filenames; + } + if (reservedCount < maxCount) { + if (wasLimited) + *wasLimited = true; + maxCount = reservedCount; + } +#endif + collectFiles(dirname, levels, maxCount, filenames, wasLimited); #endif return filenames; } diff --git a/src/FSCommon.h b/src/FSCommon.h index 2f86928af..bccbc600b 100644 --- a/src/FSCommon.h +++ b/src/FSCommon.h @@ -61,7 +61,7 @@ void fsListFiles(); bool copyFile(const char *from, const char *to); bool renameFile(const char *pathFrom, const char *pathTo); bool fsFormat(); -std::vector getFiles(const char *dirname, uint8_t levels); +std::vector getFiles(const char *dirname, uint8_t levels, size_t maxCount = 64, bool *wasLimited = nullptr); void listDir(const char *dirname, uint8_t levels, bool del = false); void rmDir(const char *dirname); void setupSDCard(); \ No newline at end of file diff --git a/src/mesh/PhoneAPI.cpp b/src/mesh/PhoneAPI.cpp index 5ee553bdc..884cbfae1 100644 --- a/src/mesh/PhoneAPI.cpp +++ b/src/mesh/PhoneAPI.cpp @@ -40,6 +40,17 @@ #include "Throttle.h" #include +namespace +{ +constexpr uint8_t FILES_MANIFEST_LEVELS = 3; +constexpr size_t FILES_MANIFEST_MAX_COUNT = 64; + +void releaseFilesManifest(std::vector &filesManifest) +{ + std::vector().swap(filesManifest); +} +} // namespace + // Flag to indicate a heartbeat was received and we should send queue status bool heartbeatReceived = false; @@ -298,18 +309,31 @@ void PhoneAPI::handleStartConfig() state = STATE_SEND_MY_INFO; } pauseBluetoothLogging = true; - spiLock->lock(); #if defined(MESHTASTIC_EXCLUDE_FILES_MANIFEST) // Skip the recursive FS walk. Used by platforms whose Zephyr LittleFS // backend can't safely traverse a deep tree (e.g. nRF54L15) and platforms // that don't support OTA browsing — the manifest is only consumed by // companion apps for those flows. - filesManifest.clear(); + releaseFilesManifest(filesManifest); #else - filesManifest = getFiles("/", 10); + // Manifest is never read on the node-info-only path (STATE_SEND_FILEMANIFEST + // short-circuits to sendConfigComplete), so skip the SPI lock + FS walk. + if (config_nonce != SPECIAL_NONCE_ONLY_NODES) { + bool filesManifestLimited = false; + { + concurrency::LockGuard guard(spiLock); + filesManifest = getFiles("/", FILES_MANIFEST_LEVELS, FILES_MANIFEST_MAX_COUNT, &filesManifestLimited); + } + if (filesManifestLimited) { + LOG_WARN("Got %zu files in manifest (limited to %zu entries/depth %u)", filesManifest.size(), + FILES_MANIFEST_MAX_COUNT, static_cast(FILES_MANIFEST_LEVELS)); + } else { + LOG_DEBUG("Got %zu files in manifest", filesManifest.size()); + } + } else { + releaseFilesManifest(filesManifest); + } #endif - spiLock->unlock(); - LOG_DEBUG("Got %d files in manifest", filesManifest.size()); LOG_INFO("Start API client config millis=%u", millis()); // Protect against concurrent BLE callbacks: they run in NimBLE's FreeRTOS task and also touch nodeInfoQueue. @@ -377,8 +401,7 @@ void PhoneAPI::close() replayPhase = REPLAY_PHASE_IDLE; } packetForPhone = NULL; - filesManifest.clear(); - filesManifest.shrink_to_fit(); + releaseFilesManifest(filesManifest); lastPortNumToRadio.clear(); fromRadioNum = 0; config_nonce = 0; @@ -943,7 +966,7 @@ size_t PhoneAPI::getFromRadio(uint8_t *buf) // ONLY_NODES variants skip the manifest. if (config_state == filesManifest.size() || config_nonce == SPECIAL_NONCE_ONLY_NODES) { config_state = 0; - filesManifest.clear(); + releaseFilesManifest(filesManifest); // Skip to complete packet sendConfigComplete(); } else { From 9e3a9c370da2b009e865aeb821ecad4c6eb1f6a8 Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Sat, 6 Jun 2026 14:15:49 -0500 Subject: [PATCH 29/86] Enhance RTC handling with unit test support for system time fallback (#10642) * Enhance RTC handling with unit test support for system time fallback * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> (cherry picked from commit c7f17a80b26a4a0ef151f5c1ba9342e7e1e205ed) --- src/gps/RTC.cpp | 99 ++++++++++++++++++++++++++++++++----- src/gps/RTC.h | 4 ++ test/test_rtc/test_main.cpp | 82 ++++++++++++++++++++++++++++++ 3 files changed, 173 insertions(+), 12 deletions(-) create mode 100644 test/test_rtc/test_main.cpp diff --git a/src/gps/RTC.cpp b/src/gps/RTC.cpp index 78439a50a..ad0bdec04 100644 --- a/src/gps/RTC.cpp +++ b/src/gps/RTC.cpp @@ -31,13 +31,63 @@ static uint32_t timeStartMsec; // Once we have a GPS lock, this is where we hold the initial msec clock that corresponds to that time static uint64_t zeroOffsetSecs; // GPS based time in secs since 1970 - only updated once on initial lock +#ifdef PIO_UNIT_TESTING +// Test seam: unit tests can inject a fake system clock (e.g. the uptime seconds that +// gettimeofday() returns on boards without a real RTC, like RP2040) and force readFromRTC() +// down the no-hardware-RTC fallback even when a hardware-RTC branch is compiled in. +static bool hasMockSystemTime = false; +static bool forceSystemTimeFallback = false; +static struct timeval mockSystemTime = {}; +#endif + +// Reads the platform system clock (or the injected mock during unit tests). Used only by the +// no-hardware-RTC fallback below, so it may be unused on builds with a hardware RTC. +[[maybe_unused]] static bool readSystemTime(struct timeval *tv) +{ +#ifdef PIO_UNIT_TESTING + if (hasMockSystemTime) { + *tv = mockSystemTime; + return true; + } +#endif + return gettimeofday(tv, NULL) == 0; +} + +// Seeds the clock from the system time on boards without a hardware RTC. gettimeofday() can +// return uptime rather than wall-clock time there (e.g. RP2040), so only adopt it when we have +// nothing better yet -- never clobber a higher-quality GPS/NTP/phone source (issue #9828). +[[maybe_unused]] static RTCSetResult readFromSystemTimeFallback() +{ + struct timeval tv; + if (readSystemTime(&tv)) { + uint32_t now = millis(); + uint32_t printableEpoch = tv.tv_sec; // Print lib only supports 32 bit but time_t can be 64 bit on some platforms + if (currentQuality == RTCQualityNone) { + LOG_DEBUG("Seed time from system clock: %lu", (unsigned long)printableEpoch); + timeStartMsec = now; + zeroOffsetSecs = tv.tv_sec; + } else { + LOG_DEBUG("Ignore system clock fallback (%lu); current RTC quality is %s", (unsigned long)printableEpoch, + RtcName(currentQuality)); + } + return RTCSetResultSuccess; + } + return RTCSetResultNotSet; +} + /** - * Reads the current date and time from the RTC module and updates the system time. - * @return True if the RTC was successfully read and the system time was updated, false otherwise. + * Reads date/time from the RTC module (or system-time fallback) and seeds internal timekeeping. + * @return RTCSetResultSuccess if a time source was read successfully (even if an existing higher-quality time is retained). */ RTCSetResult readFromRTC() { - struct timeval tv; /* btw settimeofday() is helpful here too*/ +#ifdef PIO_UNIT_TESTING + if (forceSystemTimeFallback) { + return readFromSystemTimeFallback(); + } +#endif + + [[maybe_unused]] struct timeval tv; /* btw settimeofday() is helpful here too*/ #ifdef RV3028_RTC if (rtc_found.address == RV3028_RTC) { uint32_t now = millis(); @@ -162,14 +212,7 @@ RTCSetResult readFromRTC() } } #else - if (!gettimeofday(&tv, NULL)) { - uint32_t now = millis(); - uint32_t printableEpoch = tv.tv_sec; // Print lib only supports 32 bit but time_t can be 64 bit on some platforms - LOG_DEBUG("Read RTC time as %ld", printableEpoch); - timeStartMsec = now; - zeroOffsetSecs = tv.tv_sec; - return RTCSetResultSuccess; - } + return readFromSystemTimeFallback(); #endif return RTCSetResultNotSet; } @@ -292,7 +335,7 @@ RTCSetResult perhapsSetRTC(RTCQuality q, const struct timeval *tv, bool forceUpd LOG_WARN("Failed to set time for RX8130CE"); } } -#elif defined(ARCH_ESP32) +#elif defined(ARCH_ESP32) || defined(ARCH_RP2040) settimeofday(tv, NULL); #endif @@ -423,6 +466,38 @@ void setBootRelativeTimeForUnitTest(uint32_t secondsSinceBoot) lastSetFromPhoneNtpOrGps = 0; lastTimeValidationWarning = 0; } + +void clearRTCSystemTimeForTests() +{ + hasMockSystemTime = false; + mockSystemTime = {}; +} + +void setRTCSystemTimeForTests(const struct timeval *tv) +{ + if (tv == NULL) { + clearRTCSystemTimeForTests(); + return; + } + mockSystemTime = *tv; + hasMockSystemTime = true; +} + +void setReadFromRTCUseSystemTimeForTests(bool enabled) +{ + forceSystemTimeFallback = enabled; +} + +void resetRTCStateForTests() +{ + currentQuality = RTCQualityNone; + timeStartMsec = 0; + zeroOffsetSecs = 0; + lastSetFromPhoneNtpOrGps = 0; + lastTimeValidationWarning = 0; + setReadFromRTCUseSystemTimeForTests(false); + clearRTCSystemTimeForTests(); +} #endif time_t gm_mktime(const struct tm *tm) diff --git a/src/gps/RTC.h b/src/gps/RTC.h index cd1e1d002..b69a99ec9 100644 --- a/src/gps/RTC.h +++ b/src/gps/RTC.h @@ -56,6 +56,10 @@ RTCSetResult readFromRTC(); #ifdef PIO_UNIT_TESTING void setBootRelativeTimeForUnitTest(uint32_t secondsSinceBoot); +void resetRTCStateForTests(); +void setRTCSystemTimeForTests(const struct timeval *tv); +void clearRTCSystemTimeForTests(); +void setReadFromRTCUseSystemTimeForTests(bool enabled); #endif time_t gm_mktime(const struct tm *tm); diff --git a/test/test_rtc/test_main.cpp b/test/test_rtc/test_main.cpp new file mode 100644 index 000000000..02cad01c4 --- /dev/null +++ b/test/test_rtc/test_main.cpp @@ -0,0 +1,82 @@ +#include "TestUtil.h" +#include "gps/RTC.h" +#include +#include +#include + +// Regression coverage for issue #9828: on boards without a hardware RTC (e.g. RP2040), +// gettimeofday() can return uptime seconds rather than wall-clock time. A later readFromRTC() +// must not overwrite a higher-quality network/GPS time with that value, but it should still seed +// the clock when nothing better exists yet. +// +// The native test build compiles the RV3028 hardware-RTC branch (variants/native/portduino +// defines RV3028_RTC), so these tests use setReadFromRTCUseSystemTimeForTests() to force the +// no-hardware-RTC fallback path and setRTCSystemTimeForTests() to inject a deterministic clock. + +static const uint32_t kAllowedDriftSeconds = 2; +static const time_t kUptimeSeconds = 21; // what gettimeofday() returns on RP2040 without a real clock + +// A clearly-valid wall-clock epoch, safely inside any BUILD_EPOCH validity window. +static time_t makeValidEpoch() +{ + return time(NULL) + SEC_PER_DAY; +} + +void setUp(void) +{ + resetRTCStateForTests(); +} + +void tearDown(void) +{ + resetRTCStateForTests(); +} + +// A higher-quality network time must survive a later system-time read that only knows uptime. +static void test_readFromRTC_preserves_better_network_time(void) +{ + const time_t networkEpoch = makeValidEpoch(); + struct timeval networkTime; + networkTime.tv_sec = networkEpoch; + networkTime.tv_usec = 0; + TEST_ASSERT_EQUAL_INT(RTCSetResultSuccess, perhapsSetRTC(RTCQualityFromNet, &networkTime)); + + // Simulate a later readFromRTC() falling back to a system clock that only knows uptime. + struct timeval uptime; + uptime.tv_sec = kUptimeSeconds; + uptime.tv_usec = 0; + setRTCSystemTimeForTests(&uptime); + setReadFromRTCUseSystemTimeForTests(true); + + TEST_ASSERT_EQUAL_INT(RTCSetResultSuccess, readFromRTC()); + TEST_ASSERT_EQUAL_INT(RTCQualityFromNet, getRTCQuality()); + TEST_ASSERT_UINT32_WITHIN(kAllowedDriftSeconds, (uint32_t)networkEpoch, getValidTime(RTCQualityFromNet)); +} + +// Before any higher-quality source exists, the fallback should still seed the clock. +static void test_readFromRTC_initializes_time_when_no_better_source(void) +{ + const time_t systemEpoch = makeValidEpoch(); + struct timeval systemTime; + systemTime.tv_sec = systemEpoch; + systemTime.tv_usec = 0; + setRTCSystemTimeForTests(&systemTime); + setReadFromRTCUseSystemTimeForTests(true); + + TEST_ASSERT_EQUAL_INT(RTCSetResultSuccess, readFromRTC()); + TEST_ASSERT_EQUAL_INT(RTCQualityNone, getRTCQuality()); + TEST_ASSERT_UINT32_WITHIN(kAllowedDriftSeconds, (uint32_t)systemEpoch, getTime()); +} + +void setup() +{ + delay(10); + initializeTestEnvironment(); + + UNITY_BEGIN(); + RUN_TEST(test_readFromRTC_preserves_better_network_time); + RUN_TEST(test_readFromRTC_initializes_time_when_no_better_source); + exit(UNITY_END()); +} + +void loop() {} From 245b45fff3fea8466862c928dec8783451b01d5e Mon Sep 17 00:00:00 2001 From: Austin Date: Wed, 10 Jun 2026 13:59:55 -0400 Subject: [PATCH 30/86] meshtasticd: Add configs for B&Q Station G3 (#10673) configs for raspberry pi and luckfox lyra zero Co-authored-by: Ben Meadors (cherry picked from commit bfc206a47b64738575b8039733e663218870398b) --- ...lyra-zero-BQ-SG3-BQ35LORA900V1M-slot1.yaml | 29 +++++++++++++++++++ .../lora-rpi-BQ-SG3-BQ35LORA900V1M-slot1.yaml | 17 +++++++++++ 2 files changed, 46 insertions(+) create mode 100644 bin/config.d/lora-lyra-zero-BQ-SG3-BQ35LORA900V1M-slot1.yaml create mode 100644 bin/config.d/lora-rpi-BQ-SG3-BQ35LORA900V1M-slot1.yaml diff --git a/bin/config.d/lora-lyra-zero-BQ-SG3-BQ35LORA900V1M-slot1.yaml b/bin/config.d/lora-lyra-zero-BQ-SG3-BQ35LORA900V1M-slot1.yaml new file mode 100644 index 000000000..0bee549e4 --- /dev/null +++ b/bin/config.d/lora-lyra-zero-BQ-SG3-BQ35LORA900V1M-slot1.yaml @@ -0,0 +1,29 @@ +# Station G3 + BQ35LORA900V1M Primary Slot +# Board Doc: https://wiki.bqvoy.com/en/devkits/station-g3 +Meta: + name: B&Q Station G3 + BQ35LORA900V1M Primary Slot + support: official + compatible: + - luckfox-lyra-zero-w # Armbian + +Lora: + Module: sx1262 + IRQ: # GPIO0_A5 (physical 15) + pin: 5 + gpiochip: 0 + line: 5 + Reset: # GPIO1_D1 (physical 36) + pin: 57 + gpiochip: 1 + line: 25 + Busy: # GPIO0_B4 (physical 18) + pin: 12 + gpiochip: 0 + line: 12 + DIO3_TCXO_VOLTAGE: true + DIO2_AS_RF_SWITCH: true + spidev: spidev0.0 + # CS: # GPIO0_B2 (physical 24) + # pin: 10 + # gpiochip: 0 + # line: 10 diff --git a/bin/config.d/lora-rpi-BQ-SG3-BQ35LORA900V1M-slot1.yaml b/bin/config.d/lora-rpi-BQ-SG3-BQ35LORA900V1M-slot1.yaml new file mode 100644 index 000000000..d58889517 --- /dev/null +++ b/bin/config.d/lora-rpi-BQ-SG3-BQ35LORA900V1M-slot1.yaml @@ -0,0 +1,17 @@ +# Station G3 + BQ35LORA900V1M Primary Slot +# Board Doc: https://wiki.bqvoy.com/en/devkits/station-g3 +Meta: + name: B&Q Station G3 + BQ35LORA900V1M Primary Slot + support: official + compatible: + - raspberry-pi + +Lora: + Module: sx1262 + IRQ: 22 + Reset: 16 + Busy: 24 + DIO3_TCXO_VOLTAGE: true + DIO2_AS_RF_SWITCH: true + spidev: spidev0.0 + #CS: 8 From b603ed0bbf0db0dc564c4e30053a39a1d636ad5a Mon Sep 17 00:00:00 2001 From: Littleaton <55291686+Littleaton@users.noreply.github.com> Date: Tue, 16 Jun 2026 10:22:45 -0600 Subject: [PATCH 31/86] Additional *RAK* missing values for 6421 / 13300 13302 modules + Zebra/Nebra Hat Configs (#10644) * rebased to master * Update bin/config.d/lora-ZebraHat_2W.yaml Co-authored-by: Austin * Update bin/config.d/lora-ZebraHat_1W.yaml Co-authored-by: Austin * Update bin/config.d/lora-NebraHat_2W.yaml Co-authored-by: Austin * Update bin/config.d/lora-NebraHat_1W.yaml Co-authored-by: Austin * Remove TX_GAIN_LORA configuration line * Remove TX_GAIN_LORA configuration line * Comment out TX_GAIN_LORA configuration * Comment out TX_GAIN_LORA configuration * Update lora-ok3506-RAK6421-13300-slot1.yaml * Update lora-ok3506-RAK6421-13300-slot2.yaml * Update lora-RAK6421-13300-slot1.yaml * Update lora-RAK6421-13300-slot2.yaml --------- Co-authored-by: Austin (cherry picked from commit a49729eb89007992201007792b1e728f6dabd12b) --- bin/config.d/lora-NebraHat_1W.yaml | 19 ++++++++++++++++++ bin/config.d/lora-NebraHat_2W.yaml | 20 +++++++++++++++++++ bin/config.d/lora-RAK6421-13300-slot1.yaml | 2 +- bin/config.d/lora-RAK6421-13300-slot2.yaml | 2 +- bin/config.d/lora-ZebraHat_1W.yaml | 19 ++++++++++++++++++ bin/config.d/lora-ZebraHat_2W.yaml | 20 +++++++++++++++++++ .../lora-ecb41-pge-RAK6421-13300-slot2.yaml | 2 ++ .../lora-ecb41-pge-RAK6421-13302-slot2.yaml | 2 ++ .../lora-lyra-zero-RAK6421-13300-slot2.yaml | 2 ++ .../lora-lyra-zero-RAK6421-13302-slot2.yaml | 2 ++ .../lora-ok3506-RAK6421-13300-slot2.yaml | 2 ++ .../lora-ok3506-RAK6421-13302-slot2.yaml | 2 ++ bin/config.d/lora-usb-meshtoad-e22.yaml | 2 ++ 13 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 bin/config.d/lora-NebraHat_1W.yaml create mode 100644 bin/config.d/lora-NebraHat_2W.yaml create mode 100644 bin/config.d/lora-ZebraHat_1W.yaml create mode 100644 bin/config.d/lora-ZebraHat_2W.yaml diff --git a/bin/config.d/lora-NebraHat_1W.yaml b/bin/config.d/lora-NebraHat_1W.yaml new file mode 100644 index 000000000..9f3393713 --- /dev/null +++ b/bin/config.d/lora-NebraHat_1W.yaml @@ -0,0 +1,19 @@ +# https://github.com/wehooper4/Meshtastic-Hardware/tree/main/NebraHat +# Use for 1 watt hat +Meta: + name: NebraHat 1W + support: community + compatible: + - raspberry-pi + +Lora: + Module: sx1262 # Nebra SX1262 Pi Hat - 1W + DIO2_AS_RF_SWITCH: true + DIO3_TCXO_VOLTAGE: true +# CS: 8 # Newer version of MeshtasticD do not need this? If issues uncomment this line + IRQ: 22 + Busy: 4 + Reset: 18 + RXen: 25 +I2C: + I2CDevice: /dev/i2c-1 \ No newline at end of file diff --git a/bin/config.d/lora-NebraHat_2W.yaml b/bin/config.d/lora-NebraHat_2W.yaml new file mode 100644 index 000000000..4b712dd8a --- /dev/null +++ b/bin/config.d/lora-NebraHat_2W.yaml @@ -0,0 +1,20 @@ +# https://github.com/wehooper4/Meshtastic-Hardware/tree/main/NebraHat +# Use for 2 watt hat +Meta: + name: NebraHat 2W + support: community + compatible: + - raspberry-pi + +Lora: + Module: sx1262 # Nebra SX1262 Pi Hat - 2W + DIO2_AS_RF_SWITCH: true + DIO3_TCXO_VOLTAGE: true + SX126X_MAX_POWER: 8 +# CS: 8 # Newer version of MeshtasticD do not need this? If issues uncomment this line + IRQ: 22 + Busy: 4 + Reset: 18 + RXen: 25 +I2C: + I2CDevice: /dev/i2c-1 \ No newline at end of file diff --git a/bin/config.d/lora-RAK6421-13300-slot1.yaml b/bin/config.d/lora-RAK6421-13300-slot1.yaml index a88544896..086dadbc5 100644 --- a/bin/config.d/lora-RAK6421-13300-slot1.yaml +++ b/bin/config.d/lora-RAK6421-13300-slot1.yaml @@ -18,4 +18,4 @@ Lora: DIO3_TCXO_VOLTAGE: true DIO2_AS_RF_SWITCH: true spidev: spidev0.0 - # CS: 8 \ No newline at end of file + # CS: 8 diff --git a/bin/config.d/lora-RAK6421-13300-slot2.yaml b/bin/config.d/lora-RAK6421-13300-slot2.yaml index 5fa3a7126..c0ed9017a 100644 --- a/bin/config.d/lora-RAK6421-13300-slot2.yaml +++ b/bin/config.d/lora-RAK6421-13300-slot2.yaml @@ -18,4 +18,4 @@ Lora: DIO3_TCXO_VOLTAGE: true DIO2_AS_RF_SWITCH: true spidev: spidev0.1 - # CS: 7 \ No newline at end of file + # CS: 7 diff --git a/bin/config.d/lora-ZebraHat_1W.yaml b/bin/config.d/lora-ZebraHat_1W.yaml new file mode 100644 index 000000000..f0b3c34d3 --- /dev/null +++ b/bin/config.d/lora-ZebraHat_1W.yaml @@ -0,0 +1,19 @@ +# https://github.com/wehooper4/Meshtastic-Hardware/tree/main/ZebraHAT +# Use for 1 watt hat +Meta: + name: ZebraHat 1W + support: community + compatible: + - raspberry-pi + +Lora: + Module: sx1262 # Zebra SX1262 Pi Hat - 1W + DIO2_AS_RF_SWITCH: true + DIO3_TCXO_VOLTAGE: true + SX126X_MAX_POWER: 18 + CS: 24 + IRQ: 22 + Busy: 27 + Reset: 17 +I2C: + I2CDevice: /dev/i2c-1 diff --git a/bin/config.d/lora-ZebraHat_2W.yaml b/bin/config.d/lora-ZebraHat_2W.yaml new file mode 100644 index 000000000..fe556ea77 --- /dev/null +++ b/bin/config.d/lora-ZebraHat_2W.yaml @@ -0,0 +1,20 @@ +# https://github.com/wehooper4/Meshtastic-Hardware/tree/main/ZebraHAT +# Use for 2 watt hat +Meta: + name: ZebraHat 2W + support: community + compatible: + - raspberry-pi + +Lora: + Module: sx1262 # Zebra SX1262 Pi Hat - 2W + DIO2_AS_RF_SWITCH: true + DIO3_TCXO_VOLTAGE: true + SX126X_MAX_POWER: 8 + CS: 24 + IRQ: 22 + Busy: 27 + Reset: 17 + RXen: 25 +I2C: + I2CDevice: /dev/i2c-1 diff --git a/bin/config.d/lora-ecb41-pge-RAK6421-13300-slot2.yaml b/bin/config.d/lora-ecb41-pge-RAK6421-13300-slot2.yaml index e0ef946d9..5121e8a61 100644 --- a/bin/config.d/lora-ecb41-pge-RAK6421-13300-slot2.yaml +++ b/bin/config.d/lora-ecb41-pge-RAK6421-13300-slot2.yaml @@ -29,6 +29,8 @@ Lora: - pin: 50 # GPIO1_C2 (physical 16) gpiochip: 1 line: 18 + DIO3_TCXO_VOLTAGE: true + DIO2_AS_RF_SWITCH: true spidev: spidev0.1 # CS: # GPIO0_A7 (SPI1_CSN1, physical 26) # pin: 7 diff --git a/bin/config.d/lora-ecb41-pge-RAK6421-13302-slot2.yaml b/bin/config.d/lora-ecb41-pge-RAK6421-13302-slot2.yaml index 5548bc5c7..0417d36e9 100644 --- a/bin/config.d/lora-ecb41-pge-RAK6421-13302-slot2.yaml +++ b/bin/config.d/lora-ecb41-pge-RAK6421-13302-slot2.yaml @@ -29,6 +29,8 @@ Lora: - pin: 50 # GPIO1_C2 (physical 16) gpiochip: 1 line: 18 + DIO3_TCXO_VOLTAGE: true + DIO2_AS_RF_SWITCH: true spidev: spidev0.1 # CS: # GPIO0_A7 (SPI1_CSN1, physical 26) # pin: 7 diff --git a/bin/config.d/lora-lyra-zero-RAK6421-13300-slot2.yaml b/bin/config.d/lora-lyra-zero-RAK6421-13300-slot2.yaml index 255a3eca3..63648432c 100644 --- a/bin/config.d/lora-lyra-zero-RAK6421-13300-slot2.yaml +++ b/bin/config.d/lora-lyra-zero-RAK6421-13300-slot2.yaml @@ -29,6 +29,8 @@ Lora: - pin: 13 # GPIO0_B5 gpiochip: 0 line: 13 + DIO3_TCXO_VOLTAGE: true + DIO2_AS_RF_SWITCH: true spidev: spidev0.1 # CS: # GPIO0_B1 # pin: 9 diff --git a/bin/config.d/lora-lyra-zero-RAK6421-13302-slot2.yaml b/bin/config.d/lora-lyra-zero-RAK6421-13302-slot2.yaml index 773a35ab0..d3baebd45 100644 --- a/bin/config.d/lora-lyra-zero-RAK6421-13302-slot2.yaml +++ b/bin/config.d/lora-lyra-zero-RAK6421-13302-slot2.yaml @@ -29,6 +29,8 @@ Lora: - pin: 13 # GPIO0_B5 gpiochip: 0 line: 13 + DIO3_TCXO_VOLTAGE: true + DIO2_AS_RF_SWITCH: true spidev: spidev0.1 # CS: # GPIO0_B1 # pin: 9 diff --git a/bin/config.d/lora-ok3506-RAK6421-13300-slot2.yaml b/bin/config.d/lora-ok3506-RAK6421-13300-slot2.yaml index 969c20ad3..225fb5f8d 100644 --- a/bin/config.d/lora-ok3506-RAK6421-13300-slot2.yaml +++ b/bin/config.d/lora-ok3506-RAK6421-13300-slot2.yaml @@ -31,6 +31,8 @@ Lora: - pin: 103 # GPIO3_A7 (physical 16) gpiochip: 3 line: 7 + DIO3_TCXO_VOLTAGE: true + DIO2_AS_RF_SWITCH: true spidev: spidev0.1 # CS: # GPIO0_B7 (SPI0_CSN1, physical 26) # pin: 15 diff --git a/bin/config.d/lora-ok3506-RAK6421-13302-slot2.yaml b/bin/config.d/lora-ok3506-RAK6421-13302-slot2.yaml index 36b70658b..ffb221fba 100644 --- a/bin/config.d/lora-ok3506-RAK6421-13302-slot2.yaml +++ b/bin/config.d/lora-ok3506-RAK6421-13302-slot2.yaml @@ -31,6 +31,8 @@ Lora: - pin: 103 # GPIO3_A7 (physical 16) gpiochip: 3 line: 7 + DIO3_TCXO_VOLTAGE: true + DIO2_AS_RF_SWITCH: true spidev: spidev0.1 # CS: # GPIO0_B7 (SPI0_CSN1, physical 26) # pin: 15 diff --git a/bin/config.d/lora-usb-meshtoad-e22.yaml b/bin/config.d/lora-usb-meshtoad-e22.yaml index 49182c83e..3d8b15783 100644 --- a/bin/config.d/lora-usb-meshtoad-e22.yaml +++ b/bin/config.d/lora-usb-meshtoad-e22.yaml @@ -1,3 +1,5 @@ +# This config works with all revisions of the Meshtoad USB radio. + Meta: name: meshtoad-e22 support: official From 434225eb011809afb3df1716c629850d1b490bc8 Mon Sep 17 00:00:00 2001 From: Manuel <71137295+mverch67@users.noreply.github.com> Date: Sat, 20 Jun 2026 20:15:11 +0200 Subject: [PATCH 32/86] tdeck touch driver fix (#10740) Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> (cherry picked from commit 4044f4af6c721c8808cd1aa41bf1d348027d803c) --- variants/esp32s3/t-deck/platformio.ini | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/variants/esp32s3/t-deck/platformio.ini b/variants/esp32s3/t-deck/platformio.ini index a7701549f..6ee5a9e54 100644 --- a/variants/esp32s3/t-deck/platformio.ini +++ b/variants/esp32s3/t-deck/platformio.ini @@ -55,9 +55,9 @@ build_flags = -D HAS_TFT=1 -D USE_I2S_BUZZER -D RAM_SIZE=5120 - -D LV_LVGL_H_INCLUDE_SIMPLE - -D LV_CONF_INCLUDE_SIMPLE - -D LV_COMP_CONF_INCLUDE_SIMPLE + -D LV_LVGL_H_INCLUDE_SIMPLE + -D LV_CONF_INCLUDE_SIMPLE + -D LV_COMP_CONF_INCLUDE_SIMPLE -D LV_USE_SYSMON=0 -D LV_USE_PROFILER=0 -D LV_USE_PERF_MONITOR=0 @@ -72,6 +72,7 @@ build_flags = ; -D CALIBRATE_TOUCH=0 -D LGFX_SCREEN_WIDTH=240 -D LGFX_SCREEN_HEIGHT=320 + -D LGFX_BUFSIZE=153600 -D DISPLAY_SIZE=320x240 ; landscape mode -D LGFX_DRIVER=LGFX_TDECK -D GFX_DRIVER_INC=\"graphics/LGFX/LGFX_T_DECK.h\" @@ -79,13 +80,11 @@ build_flags = ; -D GFX_DRIVER_INC=\"graphics/LVGL/LVGL_T_DECK.h\" ; -D LV_USE_ST7789=1 -D VIEW_320x240 -; -D USE_DOUBLE_BUFFER -D USE_PACKET_API -D MAP_FULL_REDRAW - -D CUSTOM_TOUCH_DRIVER +; -D CUSTOM_TOUCH_DRIVER lib_deps = ${env:t-deck.lib_deps} ${device-ui_base.lib_deps} - # renovate: datasource=github-tags depName=bb_captouch packageName=bitbank2/bb_captouch - https://github.com/bitbank2/bb_captouch/archive/refs/tags/1.3.1.zip + ;https://github.com/bitbank2/bb_captouch/archive/refs/tags/1.3.1.zip From 41cbb45cdcc34f35dd90aa7bb412bda739ef9ae7 Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Tue, 23 Jun 2026 06:03:37 -0500 Subject: [PATCH 33/86] Fix build on picomputer (cherry picked from commit 22d08b4303a9bd41872420b35fe523b5a811e4eb) --- variants/esp32s3/picomputer-s3/platformio.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/variants/esp32s3/picomputer-s3/platformio.ini b/variants/esp32s3/picomputer-s3/platformio.ini index b5a4ff178..11bae2e7c 100644 --- a/variants/esp32s3/picomputer-s3/platformio.ini +++ b/variants/esp32s3/picomputer-s3/platformio.ini @@ -37,6 +37,9 @@ extends = env:picomputer-s3 build_flags = ${env:picomputer-s3.build_flags} -D MESHTASTIC_EXCLUDE_WEBSERVER=1 + ; device-ui's I2CKeyboardScanner unconditionally calls Wire.begin(I2C_SDA, I2C_SCL); + -D I2C_SDA=8 + -D I2C_SCL=9 -D INPUTDRIVER_MATRIX_TYPE=1 -D USE_PIN_BUZZER=PIN_BUZZER -D USE_SX127x From a43cd75bd3c35313ecf41ed3409633a4c1cdfa69 Mon Sep 17 00:00:00 2001 From: Austin Date: Wed, 27 May 2026 17:31:36 -0400 Subject: [PATCH 34/86] Add Lilygo T-Impulse-Plus (#10497) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add Lilygo T-Impulse-Plus * Enable small screen layout 64x32 * trunk'd * Haptic Feedback (short and long press) * enable Charging Indicator * enable nrfutil uploads * trunk fmt * Add Lilygo T-Impulse-Plus * Enable small screen layout 64x32 * trunk'd * Haptic Feedback (short and long press) * enable Charging Indicator * enable nrfutil uploads * trunk fmt * enable proper device model * Dim the haptic duration a bit. * Fix GPS pins and speed. Module enable is active low, speed is 38400 * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * set correct geometry. * Add custom_meshtastic_* metadata to t-impulse-plus platformio.ini --------- Co-authored-by: Thomas Göttgens Co-authored-by: Ben Meadors Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> (cherry picked from commit cb867cc6c813cc870fdeff3f4b1415087f7a4a2e) --- boards/t-impulse-plus.json | 53 +++++ src/Power.cpp | 15 ++ src/input/HapticFeedback.cpp | 97 ++++++++++ src/input/HapticFeedback.h | 35 ++++ src/input/InputBroker.cpp | 11 ++ src/platform/nrf52/architecture.h | 2 + src/power/SGM41562.cpp | 109 +++++++++++ src/power/SGM41562.h | 102 ++++++++++ .../nrf52840/t-impulse-plus/platformio.ini | 19 ++ variants/nrf52840/t-impulse-plus/variant.cpp | 91 +++++++++ variants/nrf52840/t-impulse-plus/variant.h | 181 ++++++++++++++++++ 11 files changed, 715 insertions(+) create mode 100644 boards/t-impulse-plus.json create mode 100644 src/input/HapticFeedback.cpp create mode 100644 src/input/HapticFeedback.h create mode 100644 src/power/SGM41562.cpp create mode 100644 src/power/SGM41562.h create mode 100644 variants/nrf52840/t-impulse-plus/platformio.ini create mode 100644 variants/nrf52840/t-impulse-plus/variant.cpp create mode 100644 variants/nrf52840/t-impulse-plus/variant.h diff --git a/boards/t-impulse-plus.json b/boards/t-impulse-plus.json new file mode 100644 index 000000000..83b289b42 --- /dev/null +++ b/boards/t-impulse-plus.json @@ -0,0 +1,53 @@ +{ + "build": { + "arduino": { + "ldscript": "nrf52840_s140_v6.ld" + }, + "core": "nRF5", + "cpu": "cortex-m4", + "extra_flags": "-DARDUINO_NRF52840_T_IMPULSE_PLUS -DNRF52840_XXAA", + "f_cpu": "64000000L", + "hwids": [["0x239A", "0x8029"]], + "usb_product": "T-Impulse-Plus-nRF52840", + "mcu": "nrf52840", + "variant": "t-impulse-plus", + "variants_dir": "variants", + "bsp": { + "name": "adafruit" + }, + "softdevice": { + "sd_flags": "-DS140", + "sd_name": "s140", + "sd_version": "6.1.1", + "sd_fwid": "0x00B6" + }, + "bootloader": { + "settings_addr": "0xFF000" + } + }, + "connectivity": ["bluetooth"], + "debug": { + "jlink_device": "nRF52840_xxAA", + "onboard_tools": ["jlink"], + "svd_path": "nrf52840.svd" + }, + "frameworks": ["arduino"], + "name": "Lilygo T-Impulse-Plus-nRF52840", + "upload": { + "maximum_ram_size": 248832, + "maximum_size": 815104, + "require_upload_port": true, + "speed": 115200, + "protocol": "nrfutil", + "protocols": [ + "jlink", + "nrfjprog", + "nrfutil", + "stlink", + "cmsis-dap", + "blackmagic" + ] + }, + "url": "https://www.lilygo.cc/", + "vendor": "Lilygo" +} diff --git a/src/Power.cpp b/src/Power.cpp index 5b2d41567..8104b53a6 100644 --- a/src/Power.cpp +++ b/src/Power.cpp @@ -24,6 +24,7 @@ #include "main.h" #include "meshUtils.h" #include "power/PowerHAL.h" +#include "power/SGM41562.h" #include "sleep.h" #ifdef ARCH_ESP32 // #include @@ -545,6 +546,10 @@ class AnalogBatteryLevel : public HasBatteryLevel // lastly provide a fallback to indicate external power when fully charged. virtual bool isVbusIn() override { +#ifdef HAS_SGM41562 + if (sgm41562 && sgm41562->refresh()) + return sgm41562->isInputPowerGood(); +#endif #ifdef EXT_PWR_DETECT return digitalRead(EXT_PWR_DETECT) == EXT_PWR_DETECT_VALUE; @@ -561,6 +566,10 @@ class AnalogBatteryLevel : public HasBatteryLevel /// we can't be smart enough to say 'full'? virtual bool isCharging() override { +#ifdef HAS_SGM41562 + if (sgm41562 && sgm41562->refresh()) + return sgm41562->isCharging(); +#endif #if HAS_TELEMETRY && !MESHTASTIC_EXCLUDE_ENVIRONMENTAL_SENSOR && defined(HAS_RAKPROT) && !defined(HAS_PMU) if (hasRAK()) { return (rak9154Sensor.isCharging()) ? OptTrue : OptFalse; @@ -767,6 +776,12 @@ bool Power::analogInit() */ bool Power::setup() { +#ifdef HAS_SGM41562 + // Initialize the charger early so AnalogBatteryLevel can read charging + // state from it. The charger does not provide battery voltage / percent — + // those still come from the platform ADC via analogInit() below. + initSGM41562(SGM41562_WIRE); +#endif bool found = false; if (axpChipInit()) { found = true; diff --git a/src/input/HapticFeedback.cpp b/src/input/HapticFeedback.cpp new file mode 100644 index 000000000..fcd215be0 --- /dev/null +++ b/src/input/HapticFeedback.cpp @@ -0,0 +1,97 @@ +#include "HapticFeedback.h" + +#ifdef HAPTIC_FEEDBACK_PIN + +#include + +#ifdef HAPTIC_FEEDBACK_ACTIVE_LOW +#define HAPTIC_FEEDBACK_ON_STATE LOW +#define HAPTIC_FEEDBACK_OFF_STATE HIGH +#else +#define HAPTIC_FEEDBACK_ON_STATE HIGH +#define HAPTIC_FEEDBACK_OFF_STATE LOW +#endif + +HapticFeedback *hapticFeedback = nullptr; + +void initHapticFeedback() +{ + if (!hapticFeedback) + hapticFeedback = new HapticFeedback(); +} + +HapticFeedback::HapticFeedback() : concurrency::OSThread("Haptic") +{ + pinMode(HAPTIC_FEEDBACK_PIN, OUTPUT); + digitalWrite(HAPTIC_FEEDBACK_PIN, HAPTIC_FEEDBACK_OFF_STATE); +} + +void HapticFeedback::motorWrite(bool on) +{ + digitalWrite(HAPTIC_FEEDBACK_PIN, on ? HAPTIC_FEEDBACK_ON_STATE : HAPTIC_FEEDBACK_OFF_STATE); +} + +void HapticFeedback::pulse(uint16_t durationMs) +{ + motorWrite(true); + pulseOffAt = millis() + durationMs; + if (pulseOffAt == 0) // 0 is the "no pulse" sentinel + pulseOffAt = 1; + scheduleNext(); +} + +void HapticFeedback::armDelayedPulse(uint16_t delayMs, uint16_t durationMs) +{ + delayedPulseAt = millis() + delayMs; + if (delayedPulseAt == 0) + delayedPulseAt = 1; + delayedPulseDuration = durationMs; + scheduleNext(); +} + +void HapticFeedback::cancelDelayedPulse() +{ + delayedPulseAt = 0; +} + +void HapticFeedback::scheduleNext() +{ + uint32_t now = millis(); + uint32_t next = 0; + if (pulseOffAt != 0) + next = pulseOffAt; + if (delayedPulseAt != 0 && (next == 0 || (int32_t)(delayedPulseAt - next) < 0)) + next = delayedPulseAt; + if (next == 0) + return; + int32_t delay = (int32_t)(next - now); + setIntervalFromNow(delay > 0 ? (unsigned long)delay : 0); +} + +int32_t HapticFeedback::runOnce() +{ + uint32_t now = millis(); + + if (pulseOffAt != 0 && (int32_t)(now - pulseOffAt) >= 0) { + motorWrite(false); + pulseOffAt = 0; + } + + if (delayedPulseAt != 0 && (int32_t)(now - delayedPulseAt) >= 0) { + uint16_t dur = delayedPulseDuration; + delayedPulseAt = 0; + pulse(dur); + } + + uint32_t next = 0; + if (pulseOffAt != 0) + next = pulseOffAt; + if (delayedPulseAt != 0 && (next == 0 || (int32_t)(delayedPulseAt - next) < 0)) + next = delayedPulseAt; + if (next == 0) + return 60 * 1000; + int32_t delay = (int32_t)(next - now); + return delay > 0 ? delay : 0; +} + +#endif // HAPTIC_FEEDBACK_PIN diff --git a/src/input/HapticFeedback.h b/src/input/HapticFeedback.h new file mode 100644 index 000000000..da542edeb --- /dev/null +++ b/src/input/HapticFeedback.h @@ -0,0 +1,35 @@ +#pragma once + +#include "configuration.h" + +#ifdef HAPTIC_FEEDBACK_PIN + +#include "concurrency/OSThread.h" +#include + +// Non-blocking pulses on a GPIO vibration motor. HAPTIC_FEEDBACK_ACTIVE_LOW inverts polarity. +class HapticFeedback : public concurrency::OSThread +{ + public: + HapticFeedback(); + void pulse(uint16_t durationMs = 30); + void armDelayedPulse(uint16_t delayMs, uint16_t durationMs = 30); + void cancelDelayedPulse(); + + protected: + int32_t runOnce() override; + + private: + uint32_t pulseOffAt = 0; + uint32_t delayedPulseAt = 0; + uint16_t delayedPulseDuration = 0; + + void motorWrite(bool on); + // Reschedule to the soonest pending event so later arms don't clobber earlier wakes. + void scheduleNext(); +}; + +extern HapticFeedback *hapticFeedback; +void initHapticFeedback(); + +#endif // HAPTIC_FEEDBACK_PIN diff --git a/src/input/InputBroker.cpp b/src/input/InputBroker.cpp index 3d18f69cf..39c8ba77e 100644 --- a/src/input/InputBroker.cpp +++ b/src/input/InputBroker.cpp @@ -2,6 +2,7 @@ #include "PowerFSM.h" // needed for event trigger #include "configuration.h" #include "graphics/Screen.h" +#include "input/HapticFeedback.h" #include "modules/ExternalNotificationModule.h" #ifdef MESHTASTIC_LOCKDOWN #include "security/LockdownDisplay.h" @@ -256,6 +257,16 @@ void InputBroker::Init() } touchBacklightActive = false; }; +#endif +#if defined(HAPTIC_FEEDBACK_PIN) + // Blip on touch, second blip when long-press fires (500 ms = touchConfig.longPressTime default). + touchConfig.suppressLeadUpSound = true; + initHapticFeedback(); + touchConfig.onPress = []() { + hapticFeedback->pulse(80); + hapticFeedback->armDelayedPulse(500, 80); + }; + touchConfig.onRelease = []() { hapticFeedback->cancelDelayedPulse(); }; #endif TouchButtonThread->initButton(touchConfig); #endif diff --git a/src/platform/nrf52/architecture.h b/src/platform/nrf52/architecture.h index 0c04bbbab..3f14e6fa0 100644 --- a/src/platform/nrf52/architecture.h +++ b/src/platform/nrf52/architecture.h @@ -89,6 +89,8 @@ #define HW_VENDOR meshtastic_HardwareModel_T_ECHO_CARD #elif defined(TTGO_T_ECHO_PLUS) #define HW_VENDOR meshtastic_HardwareModel_T_ECHO_PLUS +#elif defined(T_IMPULSE_PLUS) +#define HW_VENDOR meshtastic_HardwareModel_T_IMPULSE_PLUS #elif defined(ELECROW_ThinkNode_M1) #define HW_VENDOR meshtastic_HardwareModel_THINKNODE_M1 #elif defined(ELECROW_ThinkNode_M3) diff --git a/src/power/SGM41562.cpp b/src/power/SGM41562.cpp new file mode 100644 index 000000000..4f6acf406 --- /dev/null +++ b/src/power/SGM41562.cpp @@ -0,0 +1,109 @@ +#include "SGM41562.h" + +#ifdef HAS_SGM41562 + +#include + +SGM41562 *sgm41562 = nullptr; + +bool initSGM41562(TwoWire &wire) +{ + if (sgm41562) + return true; + sgm41562 = new SGM41562(); + if (!sgm41562->begin(wire)) { + delete sgm41562; + sgm41562 = nullptr; + return false; + } + return true; +} + +bool SGM41562::readReg(uint8_t reg, uint8_t &value) +{ + wire_->beginTransmission(address_); + wire_->write(reg); + if (wire_->endTransmission(false) != 0) + return false; + if (wire_->requestFrom((int)address_, 1) != 1) + return false; + value = wire_->read(); + return true; +} + +bool SGM41562::writeReg(uint8_t reg, uint8_t value) +{ + wire_->beginTransmission(address_); + wire_->write(reg); + wire_->write(value); + return wire_->endTransmission() == 0; +} + +bool SGM41562::updateReg(uint8_t reg, uint8_t mask, uint8_t value) +{ + uint8_t cur; + if (!readReg(reg, cur)) + return false; + cur = (cur & ~mask) | (value & mask); + return writeReg(reg, cur); +} + +bool SGM41562::begin(TwoWire &wire, uint8_t address) +{ + wire_ = &wire; + address_ = address; + + uint8_t id; + if (!readReg(REG_DEVICE_ID, id)) { + LOG_WARN("SGM41562: I2C read failed at 0x%02X", address_); + return false; + } + if (id != DEVICE_ID_EXPECTED) { + LOG_WARN("SGM41562: unexpected device ID 0x%02X (expected 0x%02X)", id, DEVICE_ID_EXPECTED); + return false; + } + LOG_INFO("SGM41562: detected at 0x%02X (id 0x%02X)", address_, id); + + // Mirror the vendor reference init sequence: PCB OTP off, NTC off, + // watchdog off, charger enabled. These match LilyGo's stock firmware + // for the T-Impulse Plus. + delay(120); + writeReg(REG_SYS_VOLTAGE_REG, 0xB7); + writeReg(REG_MISC_OP_CONTROL, 0x40); + writeReg(REG_CHARGE_TERM_TIMER, 0x1A); + writeReg(REG_POWER_ON_CFG, 0xA4); + + return refresh(); +} + +bool SGM41562::refresh() +{ + uint32_t now = millis(); + if (lastRefreshMs_ != 0 && (now - lastRefreshMs_) < 250) + return true; // cached + lastRefreshMs_ = now == 0 ? 1 : now; + + uint8_t status, fault; + if (!readReg(REG_SYSTEM_STATUS, status)) + return false; + if (!readReg(REG_FAULT, fault)) + return false; + + chargeStatus_ = static_cast((status >> SYS_STATUS_CHRG_SHIFT) & SYS_STATUS_CHRG_MASK); + inputPowerGood_ = (status & SYS_STATUS_PG) != 0; + thermalReg_ = (status & SYS_STATUS_THERM_REG) != 0; + faultMask_ = fault & 0x3F; // bits [7:6] are enter_ship_time config, not faults + return true; +} + +bool SGM41562::setChargeEnable(bool enable) +{ + return updateReg(REG_POWER_ON_CFG, POWER_ON_CFG_CHG_DISABLE, enable ? 0x00 : POWER_ON_CFG_CHG_DISABLE); +} + +bool SGM41562::setShippingModeEnable(bool enable) +{ + return updateReg(REG_MISC_OP_CONTROL, MISC_OP_SHIPPING_MODE, enable ? MISC_OP_SHIPPING_MODE : 0x00); +} + +#endif // HAS_SGM41562 diff --git a/src/power/SGM41562.h b/src/power/SGM41562.h new file mode 100644 index 000000000..30836ff09 --- /dev/null +++ b/src/power/SGM41562.h @@ -0,0 +1,102 @@ +#pragma once + +#include "configuration.h" + +#ifdef HAS_SGM41562 + +#include +#include + +// SG Micro SGM41562 — single-cell Li-ion buck charger, I²C-controlled, no +// fuel gauge. This driver exposes status (charging / input good / fault), +// charge enable, and shipping-mode control. Battery voltage/percent still +// come from the platform ADC path; the charger is plumbed in as a +// side-channel for isCharging()/isVbusIn() in AnalogBatteryLevel. +// +// Reference: SGM41562 datasheet (Cmd map + bit fields cross-verified against +// LilyGo's `Cpp_Bus_Driver::Sgm41562xx` driver, which is what their vendor +// example for this board uses). + +#ifndef SGM41562_ADDR +#define SGM41562_ADDR 0x03 // Per datasheet — unusual but correct +#endif + +#ifndef SGM41562_WIRE +#define SGM41562_WIRE Wire1 // Most boards put the PMU on the secondary bus +#endif + +class SGM41562 +{ + public: + enum class ChargeStatus : uint8_t { + NotCharging = 0b00, + Precharge = 0b01, + FastCharge = 0b10, + ChargeDone = 0b11, + }; + + bool begin(TwoWire &wire, uint8_t address = SGM41562_ADDR); + + // Re-read the system status + fault registers. Throttled internally to + // at most one I²C transaction per 250 ms — call as often as you like. + bool refresh(); + + // Status — cached from the most recent refresh(). + ChargeStatus chargeStatus() const { return chargeStatus_; } + bool isCharging() const { return chargeStatus_ == ChargeStatus::Precharge || chargeStatus_ == ChargeStatus::FastCharge; } + bool isChargeDone() const { return chargeStatus_ == ChargeStatus::ChargeDone; } + bool isInputPowerGood() const { return inputPowerGood_; } + bool isThermalRegulation() const { return thermalReg_; } + uint8_t faultMask() const { return faultMask_; } + + // Control. + bool setChargeEnable(bool enable); + bool setShippingModeEnable(bool enable); + + private: + TwoWire *wire_ = nullptr; + uint8_t address_ = SGM41562_ADDR; + uint32_t lastRefreshMs_ = 0; + + ChargeStatus chargeStatus_ = ChargeStatus::NotCharging; + bool inputPowerGood_ = false; + bool thermalReg_ = false; + uint8_t faultMask_ = 0; + + bool readReg(uint8_t reg, uint8_t &value); + bool writeReg(uint8_t reg, uint8_t value); + bool updateReg(uint8_t reg, uint8_t mask, uint8_t value); + + // SGM41562 register addresses + static constexpr uint8_t REG_INPUT_SOURCE = 0x00; + static constexpr uint8_t REG_POWER_ON_CFG = 0x01; + static constexpr uint8_t REG_CHARGE_CURRENT = 0x02; + static constexpr uint8_t REG_DISCHARGE_TERM_CURRENT = 0x03; + static constexpr uint8_t REG_CHARGE_VOLTAGE = 0x04; + static constexpr uint8_t REG_CHARGE_TERM_TIMER = 0x05; + static constexpr uint8_t REG_MISC_OP_CONTROL = 0x06; + static constexpr uint8_t REG_SYS_VOLTAGE_REG = 0x07; + static constexpr uint8_t REG_SYSTEM_STATUS = 0x08; + static constexpr uint8_t REG_FAULT = 0x09; + static constexpr uint8_t REG_I2C_ADDR_MISC = 0x0A; + static constexpr uint8_t REG_DEVICE_ID = 0x0B; + + // Bit positions in REG_POWER_ON_CFG. + static constexpr uint8_t POWER_ON_CFG_CHG_DISABLE = 0x08; // bit 3: 1 = charging disabled + // Bit positions in REG_MISC_OP_CONTROL. + static constexpr uint8_t MISC_OP_SHIPPING_MODE = 0x20; // bit 5: 1 = enter shipping mode + // Bit positions in REG_SYSTEM_STATUS. + static constexpr uint8_t SYS_STATUS_CHRG_SHIFT = 3; + static constexpr uint8_t SYS_STATUS_CHRG_MASK = 0x03; + static constexpr uint8_t SYS_STATUS_PG = 0x02; // bit 1: input power good + static constexpr uint8_t SYS_STATUS_THERM_REG = 0x01; // bit 0: thermal regulation + + static constexpr uint8_t DEVICE_ID_EXPECTED = 0x04; +}; + +extern SGM41562 *sgm41562; + +// Lazy-instantiate the global on the supplied wire. Returns true on success. +bool initSGM41562(TwoWire &wire); + +#endif // HAS_SGM41562 diff --git a/variants/nrf52840/t-impulse-plus/platformio.ini b/variants/nrf52840/t-impulse-plus/platformio.ini new file mode 100644 index 000000000..337b586dc --- /dev/null +++ b/variants/nrf52840/t-impulse-plus/platformio.ini @@ -0,0 +1,19 @@ +[env:t-impulse-plus] +custom_meshtastic_hw_model = 135 +custom_meshtastic_hw_model_slug = T_IMPULSE_PLUS +custom_meshtastic_architecture = nrf52840 +custom_meshtastic_actively_supported = true +custom_meshtastic_support_level = 1 +custom_meshtastic_display_name = LILYGO T-Impulse Plus +custom_meshtastic_tags = LilyGo +custom_meshtastic_requires_dfu = true + +extends = nrf52840_base +board = t-impulse-plus +board_level = pr + +build_flags = ${nrf52840_base.build_flags} + -I variants/nrf52840/t-impulse-plus + -D T_IMPULSE_PLUS + +build_src_filter = ${nrf52_base.build_src_filter} +<../variants/nrf52840/t-impulse-plus> diff --git a/variants/nrf52840/t-impulse-plus/variant.cpp b/variants/nrf52840/t-impulse-plus/variant.cpp new file mode 100644 index 000000000..c50a2527d --- /dev/null +++ b/variants/nrf52840/t-impulse-plus/variant.cpp @@ -0,0 +1,91 @@ +/* + * variant.cpp - Digital pin mapping for LilyGo T-Impulse Plus + * + * Board: T-Impulse Plus V1.0 (nRF52840) + * Hardware: + * - SSD1315 OLED + * - SX1262 (S62F) + * - MIA-M10Q GPS + * - ICM20948 IMU + * - ZD25WQ32C Flash + * - TTP223 Touch Button + * - Vibration Motor + */ + +#include "variant.h" +#include "nrf.h" +#include "wiring_constants.h" +#include "wiring_digital.h" + +extern "C" { +const uint32_t g_ADigitalPinMap[] = { + // D0-D6: LoRa SX1262 (S62F module) SPI + control + 2, // D0 P0.02 SX1262_RST + 29, // D1 P0.29 SX1262_DIO1 + 31, // D2 P0.31 SX1262_BUSY + 46, // D3 P1.14 SX1262_CS + 3, // D4 P0.03 SPI_SCK + 30, // D5 P0.30 SPI_MISO + 28, // D6 P0.28 SPI_MOSI + + // D7-D8: RF switch control + 45, // D7 P1.13 SX1262_RF_VC1 (TXEN) + 39, // D8 P1.07 SX1262_RF_VC2 (RXEN) + + // D9-D11: GPS (u-blox MIA-M10Q) + 44, // D9 P1.12 GPS_TX (MCU TX -> GPS RX) + 43, // D10 P1.11 GPS_RX (MCU RX <- GPS TX) + 42, // D11 P1.10 GPS_EN + + // D12-D13: Display I2C (SSD1315) + 20, // D12 P0.20 SCREEN_SDA + 15, // D13 P0.15 SCREEN_SCL + + // D14-D15: Sensor I2C (ICM20948, SGM41562) + 40, // D14 P1.08 IMU_SDA + 11, // D15 P0.11 IMU_SCL + + // D16-D17: Battery management + 5, // D16 P0.05 BATTERY_ADC + 25, // D17 P0.25 BATTERY_MEASUREMENT_CONTROL + + // D18: Touch button (TTP223) + 36, // D18 P1.04 TTP223_KEY + + // D19: Vibration motor + 22, // D19 P0.22 VIBRATION_MOTOR + + // D20: LDO enable + 14, // D20 P0.14 RT9080_EN + + // D21-D26: Flash QSPI (ZD25WQ32C) + 12, // D21 P0.12 FLASH_CS + 4, // D22 P0.04 FLASH_SCLK + 6, // D23 P0.06 FLASH_IO0 + 41, // D24 P1.09 FLASH_IO1 + 8, // D25 P0.08 FLASH_IO2 + 26, // D26 P0.26 FLASH_IO3 + + // D27-D28: Interrupt lines + 7, // D27 P0.07 ICM20948_INT + 16, // D28 P0.16 SGM41562_INT + + // D29: Boot button + 24, // D29 P0.24 BOOT +}; +} + +void initVariant() +{ + // Flash CS high (deselect) + pinMode(PIN_QSPI_CS, OUTPUT); + digitalWrite(PIN_QSPI_CS, HIGH); + + // Enable battery voltage measurement + pinMode(BAT_READ, OUTPUT); + digitalWrite(BAT_READ, HIGH); + + // Enable RT9080 LDO + pinMode(D20, OUTPUT); + digitalWrite(D20, HIGH); +} \ No newline at end of file diff --git a/variants/nrf52840/t-impulse-plus/variant.h b/variants/nrf52840/t-impulse-plus/variant.h new file mode 100644 index 000000000..ff3300956 --- /dev/null +++ b/variants/nrf52840/t-impulse-plus/variant.h @@ -0,0 +1,181 @@ +#ifndef _T_IMPULSE_PLUS_H_ +#define _T_IMPULSE_PLUS_H_ +#include "WVariant.h" + +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +// Clock Configuration +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +#define VARIANT_MCK (64000000ul) +#define USE_LFXO + +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +// Pin Capacity Definitions +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +#define PINS_COUNT (30u) +#define NUM_DIGITAL_PINS (30u) +#define NUM_ANALOG_INPUTS (1u) +#define NUM_ANALOG_OUTPUTS (0u) + +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +// Digital Pin Mapping +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +#define D0 0 // P0.02 SX1262_RST +#define D1 1 // P0.29 SX1262_DIO1 +#define D2 2 // P0.31 SX1262_BUSY +#define D3 3 // P1.14 SX1262_CS +#define D4 4 // P0.03 SPI_SCK +#define D5 5 // P0.30 SPI_MISO +#define D6 6 // P0.28 SPI_MOSI +#define D7 7 // P1.13 RF_VC1 (TXEN) +#define D8 8 // P1.07 RF_VC2 (RXEN) +#define D9 9 // P1.12 GPS module TX → MCU RX +#define D10 10 // P1.11 GPS module RX ← MCU TX +#define D11 11 // P1.10 GPS_EN (active LOW) +#define D12 12 // P0.20 SCREEN_SDA +#define D13 13 // P0.15 SCREEN_SCL +#define D14 14 // P1.08 IMU_SDA +#define D15 15 // P0.11 IMU_SCL +#define D16 16 // P0.05 BATTERY_ADC +#define D17 17 // P0.25 BATTERY_CTL +#define D18 18 // P1.04 TTP223_KEY +#define D19 19 // P0.22 VIBRATION_MOTOR +#define D20 20 // P0.14 RT9080_EN +#define D21 21 // P0.12 FLASH_CS +#define D22 22 // P0.04 FLASH_SCLK +#define D23 23 // P0.06 FLASH_IO0 +#define D24 24 // P1.09 FLASH_IO1 +#define D25 25 // P0.08 FLASH_IO2 +#define D26 26 // P0.26 FLASH_IO3 +#define D27 27 // P0.07 ICM20948_INT +#define D28 28 // P0.16 SGM41562_INT +#define D29 29 // P0.24 BOOT + +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +// LED Configuration (no physical LED) +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +#define LED_STATE_ON 1 + +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +// Button Configuration (TTP223 capacitive touch) +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +#define PIN_BUTTON_TOUCH D18 +#define BUTTON_TOUCH_ACTIVE_LOW true +#define BUTTON_TOUCH_ACTIVE_PULLUP false + +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +// Analog Pin Definitions +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +#define PIN_VBAT D16 // P0.05 Battery voltage sense + +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +// I2C Configuration +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +// Primary I2C: Display (SSD1315) +#define PIN_WIRE_SDA D12 // P0.20 +#define PIN_WIRE_SCL D13 // P0.15 + +// Secondary I2C: IMU (ICM20948) + PMU (SGM41562) +#define WIRE_INTERFACES_COUNT 2 +#define PIN_WIRE1_SDA D14 // P1.08 +#define PIN_WIRE1_SCL D15 // P0.11 +#define I2C_NO_RESCAN + +static const uint8_t SDA = PIN_WIRE_SDA; +static const uint8_t SCL = PIN_WIRE_SCL; + +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +// Display (SSD1315, compatible with SSD1306 driver) +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +#define HAS_SCREEN 1 +#define USE_SSD1306 1 +#define OLED_TINY +#define OLED_GEOMETRY_OVERRIDE GEOMETRY_64_32 + +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +// SPI Configuration (SX1262 LoRa) +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +#define SPI_INTERFACES_COUNT 1 +#define PIN_SPI_SCK D4 // P0.03 +#define PIN_SPI_MISO D5 // P0.30 +#define PIN_SPI_MOSI D6 // P0.28 + +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +// SX1262 LoRa (S62F module) +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +#define USE_SX1262 +#define SX126X_CS D3 +#define SX126X_DIO1 D1 +#define SX126X_BUSY D2 +#define SX126X_RESET D0 +#define SX126X_DIO3_TCXO_VOLTAGE 1.8 +#define SX126X_TXEN D7 // RF_VC1 +#define SX126X_RXEN D8 // RF_VC2 + +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +// Power Management +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +#define BAT_READ D17 // P0.25 Battery measurement control (HIGH = enable) +#define BATTERY_PIN PIN_VBAT +#define BATTERY_SENSE_RESOLUTION_BITS 12 +#define ADC_MULTIPLIER 2.0 +#define AREF_VOLTAGE 3.6 + +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +// GPS (u-blox MIA-M10Q) +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +#define GPS_UBLOX +#define HAS_GPS 1 +#define GPS_RX_PIN D9 // P1.12 — MCU RX, wired to GPS module TX +#define GPS_TX_PIN D10 // P1.11 — MCU TX, wired to GPS module RX +#define PIN_GPS_EN D11 // P1.10 +#define GPS_EN_ACTIVE LOW +#define GPS_BAUDRATE 38400 +#define GPS_THREAD_INTERVAL 50 +#define PIN_SERIAL1_TX GPS_TX_PIN +#define PIN_SERIAL1_RX GPS_RX_PIN + +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +// On-board QSPI Flash (ZD25WQ32CEIGR) +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +#define PIN_QSPI_SCK D22 // P0.04 +#define PIN_QSPI_CS D21 // P0.12 +#define PIN_QSPI_IO0 D23 // P0.06 +#define PIN_QSPI_IO1 D24 // P1.09 +#define PIN_QSPI_IO2 D25 // P0.08 +#define PIN_QSPI_IO3 D26 // P0.26 + +#define EXTERNAL_FLASH_DEVICES W25Q32JV_IQ +#define EXTERNAL_FLASH_USE_QSPI + +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +// Vibration Motor (GPIO active-high) +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +#define LED_NOTIFICATION D19 // P0.22 +#define HAPTIC_FEEDBACK_PIN LED_NOTIFICATION + +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +// IMU (ICM20948 on Wire1) +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +#define HAS_ICM20948 + +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +// Charger (SGM41562 on Wire1 @ 0x03) +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +#define HAS_SGM41562 +#define SGM41562_WIRE Wire1 + +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +// Compatibility Definitions +// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +#ifdef __cplusplus +extern "C" { +#endif + +#define PIN_SERIAL2_RX (-1) +#define PIN_SERIAL2_TX (-1) + +#ifdef __cplusplus +} +#endif + +#endif // _T_IMPULSE_PLUS_H_ \ No newline at end of file From 3c68c2eed9a1e06a86b88e85dd95f4ffe313821d Mon Sep 17 00:00:00 2001 From: Austin Date: Tue, 2 Jun 2026 18:12:27 -0400 Subject: [PATCH 35/86] Debian: Correctly build without signing (for forks) (#10605) (cherry picked from commit 266c143359273fd4303989b650b297fd3481f093) --- debian/ci_pack_sdeb.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/ci_pack_sdeb.sh b/debian/ci_pack_sdeb.sh index d35aeef24..9aa018257 100755 --- a/debian/ci_pack_sdeb.sh +++ b/debian/ci_pack_sdeb.sh @@ -33,5 +33,5 @@ if [[ -n $GPG_KEY_ID ]]; then debuild -S -nc -k"$GPG_KEY_ID" else # Build the source deb without signing (forks) - debuild -S -nc + debuild -S -nc -us -uc fi From 90f01c2b95071ea666a9033f65e0f62f772fff87 Mon Sep 17 00:00:00 2001 From: nomdetom Date: Wed, 17 Jun 2026 12:32:05 +0100 Subject: [PATCH 36/86] docs: update test instructions to prefer bin/run-tests.sh; fix suite count to 19; add Copilot interface caveat --- .github/copilot-instructions.md | 28 ++++++++++++++++++++++++---- AGENTS.md | 26 +++++++++++++------------- test/README.md | 16 +++++++++++++++- 3 files changed, 52 insertions(+), 18 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 404eb021c..0b566acb3 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -635,27 +635,47 @@ Most workflows can be triggered manually via `workflow_dispatch` for testing. ### Native unit tests (C++) -Unit tests in `test/` directory with 17 test suites: +Unit tests in `test/` directory with 19 test suites: - `test_admin_radio/` - LoRa region/config validation and AdminModule dispatch - `test_atak/` - ATAK integration - `test_crypto/` - Cryptography - `test_default/` - Default configuration +- `test_hop_scaling/` - Hop scaling histogram and required-hop logic - `test_http_content_handler/` - HTTP handling - `test_mac_from_string/` - MAC address parsing - `test_mesh_module/` - Module framework - `test_meshpacket_serializer/` - Packet serialization - `test_mqtt/` - MQTT integration - `test_packet_history/` - Packet history tracking +- `test_packet_signing/` - Packet signing - `test_position_precision/` - Position precision helpers - `test_radio/` - Radio interface - `test_serial/` - Serial communication -- `test_traffic_management/` - Traffic management +- `test_traffic_management/` - Traffic management (dedup, rate-limit, hop-trim, role exceptions) - `test_transmit_history/` - Retransmission tracking - `test_type_conversions/` - NodeDB v25 type conversion (bitfield round-trips, NodeInfoLite) - `test_utf8/` - UTF-8 utilities -Run command (preferred — avoids pipe-buffering and the Ubuntu externally-managed-environment error): +**Preferred run command — `bin/run-tests.sh`** (uses the `coverage` env with ASan/LSan sanitizers, emits an unambiguous RED/AMBER/GREEN verdict, and catches missing suites as AMBER): + +```bash +./bin/run-tests.sh # all suites +./bin/run-tests.sh -f test_traffic_management # single suite +./bin/run-tests.sh -f test_traffic_management > /tmp/test_out.txt 2>&1; tail -5 /tmp/test_out.txt +``` + +Exit codes: 0 = GREEN, 1 = RED, 2 = AMBER. The final line is machine-readable: + +``` +RESULT: GREEN 19/19 suites passed +RESULT: RED test_traffic_management: 1 failed +RESULT: AMBER 18/19 suites ran (missing: test_radio) — all that ran passed +``` + +> **Copilot interface note:** When running tests via the Copilot chat interface, edits made through the chat may not be reflected in the on-disk files that the test binary reads. If tests pass in chat but fail locally (or vice versa), verify the files on disk match what you expect before trusting the result. Always confirm with a local terminal run. + +Raw `pio test` (no sanitizers, no verdict logic) — use only when you need to override the env: ```bash ~/.platformio/penv/bin/python -m platformio test -e native -f test_your_suite > /tmp/test_out.txt 2>&1 @@ -663,7 +683,7 @@ grep -E 'error:|PASS|FAIL|succeeded|failed' /tmp/test_out.txt tail -15 /tmp/test_out.txt ``` -Do **not** use `pio test … | tail -N` — it discards build errors and shows stale cached results. Do **not** use `pio test … | grep` — line-buffering makes the terminal appear hung until the process exits. Redirect to a file first, then grep. +Do **not** pipe `pio test` — line-buffering makes the terminal appear hung and hides build errors. Simulation testing: `bin/test-simulator.sh` diff --git a/AGENTS.md b/AGENTS.md index 04c0009ab..287ce95a4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,18 +10,18 @@ This file (`AGENTS.md`) is a short pointer + quick reference for agents that don ## Quick command reference -| Action | Command | -| -------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Build a firmware variant | `pio run -e ` (e.g. `pio run -e rak4631`, `pio run -e heltec-v3`) | -| Build native macOS host binary | `pio run -e native-macos` (Homebrew prereqs + CH341 LoRa setup in `variants/native/portduino/platformio.ini`) | -| Clean + rebuild | `pio run -e -t clean && pio run -e ` | -| Flash a device | `pio run -e -t upload --upload-port ` (or use the `pio_flash` MCP tool) | -| Run firmware unit tests (native) | `~/.platformio/penv/bin/python -m platformio test -e native > /tmp/test_out.txt 2>&1` then `grep -E 'error:\|PASS\|FAIL\|succeeded\|failed' /tmp/test_out.txt` (redirect first — piping causes line-buffering) | -| Run MCP hardware tests | `./mcp-server/run-tests.sh` | -| Live TUI test runner | `mcp-server/.venv/bin/meshtastic-mcp-test-tui` | -| Format before commit | `trunk fmt` | -| Regenerate protobuf bindings | `bin/regen-protos.sh` | -| Generate CI matrix | `./bin/generate_ci_matrix.py all [--level pr]` | +| Action | Command | +| -------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Build a firmware variant | `pio run -e ` (e.g. `pio run -e rak4631`, `pio run -e heltec-v3`) | +| Build native macOS host binary | `pio run -e native-macos` (Homebrew prereqs + CH341 LoRa setup in `variants/native/portduino/platformio.ini`) | +| Clean + rebuild | `pio run -e -t clean && pio run -e ` | +| Flash a device | `pio run -e -t upload --upload-port ` (or use the `pio_flash` MCP tool) | +| Run firmware unit tests (native) | `./bin/run-tests.sh` (preferred — ASan/LSan + RED/AMBER/GREEN verdict); or raw: `~/.platformio/penv/bin/python -m platformio test -e native > /tmp/test_out.txt 2>&1` | +| Run MCP hardware tests | `./mcp-server/run-tests.sh` | +| Live TUI test runner | `mcp-server/.venv/bin/meshtastic-mcp-test-tui` | +| Format before commit | `trunk fmt` | +| Regenerate protobuf bindings | `bin/regen-protos.sh` | +| Generate CI matrix | `./bin/generate_ci_matrix.py all [--level pr]` | ## MCP server (device + test automation) @@ -108,7 +108,7 @@ Sequence these; don't parallelize on the same port. | `src/modules/` | Feature modules; `Telemetry/Sensor/` has 50+ I2C sensor drivers | | `variants/` | 200+ hardware variant definitions (`variant.h` + `platformio.ini` per board) | | `protobufs/` | `.proto` definitions; regenerate with `bin/regen-protos.sh` | -| `test/` | Firmware unit tests (17 suites; `pio test -e native`) | +| `test/` | Firmware unit tests (19 suites; `./bin/run-tests.sh` preferred, falls back to `pio test -e native`) | | `mcp-server/` | Python MCP server + pytest hardware integration tests | | `mcp-server/tests/` | Tiered pytest suite: `unit/`, `mesh/`, `telemetry/`, `monitor/`, `recovery/`, `ui/`, `fleet/`, `admin/`, `provisioning/` | | `.claude/commands/` | Claude Code slash command bodies | diff --git a/test/README.md b/test/README.md index b9f6b961a..18eb68403 100644 --- a/test/README.md +++ b/test/README.md @@ -4,6 +4,20 @@ This directory contains C++ unit tests that run on the host machine via Platform ## Running Tests +**Preferred: use `bin/run-tests.sh`** — it runs the `coverage` env (ASan/LSan sanitizers), cross-checks the number of suites that actually ran, and emits an unambiguous RED/AMBER/GREEN verdict: + +```bash +./bin/run-tests.sh # all suites +./bin/run-tests.sh -f test_traffic_management # single suite +./bin/run-tests.sh -f test_traffic_management > /tmp/test_out.txt 2>&1; tail -5 /tmp/test_out.txt +``` + +Exit codes: 0 = GREEN, 1 = RED, 2 = AMBER. + +> **Copilot interface note:** When running tests via the Copilot chat interface, edits made through the chat may not be reflected in the on-disk files that the test binary reads. If tests pass in chat but fail locally (or vice versa), verify the files on disk match what you expect before trusting the result. Always confirm with a local terminal run. + +**Raw `pio test` (no sanitizers, no verdict logic)** — use when you need to override the env or inspect verbose Unity output: + ```bash # All test suites pio test -e native @@ -17,7 +31,7 @@ pio test -e native -f test_your_module -vvv **Never pipe through `| tail -N` to shorten output.** PlatformIO prints build errors at the top of output and test results at the bottom; `tail` will show stale cached results from a prior successful build while hiding the compile error that caused the current run to fail. -**Preferred pattern — redirect to file, then grep:** +**Preferred pattern for raw pio — redirect to file, then grep:** ```bash # Redirect all output to a file; grep for errors and results after it exits From 0094ad0444b91650183de8d3dbff1cd1014aeb4f Mon Sep 17 00:00:00 2001 From: nomdetom Date: Thu, 25 Jun 2026 13:39:03 +0100 Subject: [PATCH 37/86] address copilot review --- .github/copilot-instructions.md | 69 +++++++++++++++++++++----- AGENTS.md | 16 ++++++ CLAUDE.md | 21 ++++++++ bin/run-tests.sh | 88 ++++++++++++++++++++++++++++----- test/native-suite-count | 1 + 5 files changed, 169 insertions(+), 26 deletions(-) create mode 100644 CLAUDE.md create mode 100644 test/native-suite-count diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 0b566acb3..d135e46fc 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -1,5 +1,21 @@ # Meshtastic Firmware - Copilot Instructions +> **TL;DR** +> | | | +> |---|---| +> | Local tests | `./bin/run-tests.sh` (exit 0 GREEN · 1 RED · 2 AMBER · 3 FILTERED) | +> | Hardware tests | `./mcp-server/run-tests.sh` | +> | Format | `trunk fmt` | +> | Mirror docs | `AGENTS.md` (short pointer for agents that don't read this file) · `CLAUDE.md` (Claude Code) | +> +> **Need this? It's here.** +> | | | +> |---|---| +> | General helpers (clamp, UTF-8, string fmt…) | `src/meshUtils.h` | +> | Logging macros (LOG_DEBUG / INFO / WARN…) | `src/DebugConfiguration.h` | +> | New module skeleton | inherit `ProtobufModule` in `src/mesh/ProtobufModule.h` | +> | Observer / event wiring | `src/Observer.h` | + This document provides context and guidelines for AI assistants working with the Meshtastic firmware codebase. ## Project Overview @@ -37,7 +53,7 @@ MQTT provides a bridge between Meshtastic mesh networks and the internet, enabli Messages are published/subscribed using a hierarchical topic format: -``` +```text {root}/{channel_id}/{gateway_id} ``` @@ -247,7 +263,7 @@ Unit tests for the conversion layer live in `test/test_type_conversions/test_mai ## Project Structure -``` +```text firmware/ ├── src/ # Main source code │ ├── main.cpp # Application entry point @@ -482,7 +498,7 @@ Key defines in variant.h: ## Build System -## Agent Tooling Baseline +### Agent Tooling Baseline Mirror counterpart: `AGENTS.md` under **Agent Tooling Baseline**. @@ -635,7 +651,7 @@ Most workflows can be triggered manually via `workflow_dispatch` for testing. ### Native unit tests (C++) -Unit tests in `test/` directory with 19 test suites: +Unit tests in `test/` directory. The canonical suite count is in `test/native-suite-count` and is cross-checked on every full run. Current suites: - `test_admin_radio/` - LoRa region/config validation and AdminModule dispatch - `test_atak/` - ATAK integration @@ -647,30 +663,57 @@ Unit tests in `test/` directory with 19 test suites: - `test_mesh_module/` - Module framework - `test_meshpacket_serializer/` - Packet serialization - `test_mqtt/` - MQTT integration +- `test_nexthop_routing/` - Next-hop routing logic +- `test_nodedb_blocked/` - NodeDB blocked-node handling - `test_packet_history/` - Packet history tracking - `test_packet_signing/` - Packet signing +- `test_position_module/` - Position module behaviour - `test_position_precision/` - Position precision helpers - `test_radio/` - Radio interface +- `test_rtc/` - RTC / time handling - `test_serial/` - Serial communication - `test_traffic_management/` - Traffic management (dedup, rate-limit, hop-trim, role exceptions) - `test_transmit_history/` - Retransmission tracking - `test_type_conversions/` - NodeDB v25 type conversion (bitfield round-trips, NodeInfoLite) - `test_utf8/` - UTF-8 utilities +- `test_warm_store/` - Warm-tier node store -**Preferred run command — `bin/run-tests.sh`** (uses the `coverage` env with ASan/LSan sanitizers, emits an unambiguous RED/AMBER/GREEN verdict, and catches missing suites as AMBER): +**Preferred run command — `bin/run-tests.sh`** (uses the `coverage` env with ASan/LSan sanitizers; emits a machine-readable verdict on the final line; update `test/native-suite-count` when adding or removing suites): ```bash -./bin/run-tests.sh # all suites -./bin/run-tests.sh -f test_traffic_management # single suite -./bin/run-tests.sh -f test_traffic_management > /tmp/test_out.txt 2>&1; tail -5 /tmp/test_out.txt +./bin/run-tests.sh # all suites +./bin/run-tests.sh -f test_traffic_management # single suite (yields FILTERED, not GREEN) ``` -Exit codes: 0 = GREEN, 1 = RED, 2 = AMBER. The final line is machine-readable: +Exit codes and verdicts (exact counts will vary; examples below are illustrative): -``` -RESULT: GREEN 19/19 suites passed -RESULT: RED test_traffic_management: 1 failed -RESULT: AMBER 18/19 suites ran (missing: test_radio) — all that ran passed +| Exit | Verdict | Meaning | +| ---- | ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 0 | `GREEN` | All canonical suites ran, all passed, no ignored test cases | +| 1 | `RED` | At least one failure, build error, or sanitizer fault | +| 2 | `AMBER` | All that ran passed, but something was lost: a suite silently went missing on a full run, individual test cases were skipped (`TEST_IGNORE`), or `test/native-suite-count` disagrees with the `test/` directory count | +| 3 | `FILTERED` | A `-f` run completed cleanly; suites outside the filter were intentionally not run | + +Examples — exact counts will vary by suite count and env: + +```text +# GREEN: all suites ran and passed +RESULT: GREEN N/N suites passed [canonical: N/N] + +# RED: real test failure +RESULT: RED 1 failed + +# RED: sanitizer exit-time abort (all tests passed but process aborted at exit) +RESULT: RED exit-time abort (tests passed; likely sanitizer — see hint above) + +# AMBER: native-suite-count disagrees with test/ directory count (too low) +RESULT: AMBER test/ has 24 suite directories but native-suite-count says 5 — update test/native-suite-count after registering new suites + +# AMBER: native-suite-count disagrees with test/ directory count (too high) +RESULT: AMBER test/ has 24 suite directories but native-suite-count says 99 — update test/native-suite-count after removing suites + +# FILTERED: single suite run completed cleanly +RESULT: FILTERED 1/24 suites ran (not run: test_admin_radio test_atak …) — filtered: test_serial [canonical: 1/24] ``` > **Copilot interface note:** When running tests via the Copilot chat interface, edits made through the chat may not be reflected in the on-disk files that the test binary reads. If tests pass in chat but fail locally (or vice versa), verify the files on disk match what you expect before trusting the result. Always confirm with a local terminal run. diff --git a/AGENTS.md b/AGENTS.md index 287ce95a4..2124f65f4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,21 @@ # Agent instructions +> **TL;DR** +> | | | +> |---|---| +> | Local tests | `./bin/run-tests.sh` (exit 0 GREEN · 1 RED · 2 AMBER · 3 FILTERED) | +> | Hardware tests | `./mcp-server/run-tests.sh` | +> | Format | `trunk fmt` | +> | Mirror docs | `.github/copilot-instructions.md` (canonical) · `CLAUDE.md` (Claude Code) | +> +> **Need this? It's here.** +> | | | +> |---|---| +> | General helpers (clamp, UTF-8, string fmt…) | `src/meshUtils.h` | +> | Logging macros (LOG_DEBUG / INFO / WARN…) | `src/DebugConfiguration.h` | +> | New module skeleton | inherit `ProtobufModule` in `src/mesh/ProtobufModule.h` | +> | Observer / event wiring | `src/Observer.h` | + This repository is the [Meshtastic](https://meshtastic.org) firmware — a C++17 embedded codebase targeting ESP32 / nRF52 / RP2040 / STM32WL / Linux-Portduino LoRa mesh radios — plus a Python MCP server in `mcp-server/` that AI agents use to flash, configure, and test connected devices. ## Primary instruction file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..3b24bf807 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,21 @@ +# Claude Code instructions + +> **TL;DR** +> | | | +> |---|---| +> | Local tests | `./bin/run-tests.sh` (exit 0 GREEN · 1 RED · 2 AMBER · 3 FILTERED) | +> | Hardware tests | `./mcp-server/run-tests.sh` | +> | Format | `trunk fmt` | +> | Mirror docs | `.github/copilot-instructions.md` (canonical) · `AGENTS.md` | +> +> **Need this? It's here.** +> | | | +> |---|---| +> | General helpers (clamp, UTF-8, string fmt…) | `src/meshUtils.h` | +> | Logging macros (LOG_DEBUG / INFO / WARN…) | `src/DebugConfiguration.h` | +> | New module skeleton | inherit `ProtobufModule` in `src/mesh/ProtobufModule.h` | +> | Observer / event wiring | `src/Observer.h` | + +**Read `.github/copilot-instructions.md` first.** That file is the canonical agent-facing document for this repo. It covers project layout, coding conventions, the build system, CI/CD, the native C++ test suite, and the MCP Server & Hardware Test Harness. Read it top-to-bottom before starting any non-trivial change. + +This file (`CLAUDE.md`) is a short pointer for Claude Code sessions. Slash commands live in `.claude/commands/`. diff --git a/bin/run-tests.sh b/bin/run-tests.sh index 5965999cd..de824ad82 100755 --- a/bin/run-tests.sh +++ b/bin/run-tests.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Run native PlatformIO unit tests and emit a single, unambiguous RED/AMBER/GREEN verdict. +# Run native PlatformIO unit tests and emit a single, unambiguous verdict. # # Why this exists: PlatformIO reports failures three different ways ([FAILED], :FAIL:, # [ERRORED]) and an all-pass run prints "N succeeded" with NO "0 failed" clause — so naive @@ -8,16 +8,26 @@ # canonical set in test/ so a suite silently going missing shows up as AMBER, not green. # # Usage: -# ./bin/run-tests.sh # run all suites, full RAG + count cross-check -# ./bin/run-tests.sh -f test_utf8 # run one suite (no count cross-check) +# ./bin/run-tests.sh # run all suites, full verdict + count cross-check +# ./bin/run-tests.sh -f test_utf8 # run one suite (yields FILTERED, not GREEN) # ./bin/run-tests.sh -e native # override env (default: coverage) # ./bin/run-tests.sh --quiet # only print the final RESULT line # -# Exit codes: 0 = GREEN, 1 = RED, 2 = AMBER. +# Exit codes: 0 = GREEN, 1 = RED, 2 = AMBER, 3 = FILTERED. +# +# Verdicts: +# GREEN — all canonical suites ran, all passed, no ignored test cases. +# AMBER — all that ran passed, but something was lost: a suite silently went missing on a +# full run, or individual test cases were skipped (Unity TEST_IGNORE / :IGNORE:). +# FILTERED — a -f run completed cleanly; suites not in the filter were intentionally skipped. +# Use this when iterating on a single suite; it is not a quality signal. +# RED — at least one failure, build error, or sanitizer fault. # # The final line is machine-readable, e.g.: -# RESULT: GREEN 19/19 suites passed -# RESULT: AMBER 17/19 suites ran (missing: test_radio test_serial) — all that ran passed +# RESULT: GREEN N/N suites passed +# RESULT: AMBER N/M suites ran (missing: test_radio test_serial) — all that ran passed +# RESULT: AMBER 3 test case(s) ignored +# RESULT: FILTERED 1/N suites ran (not run: …) — filtered: test_utf8 # RESULT: RED test_traffic_management: 1 failed (or: build/crash error) # RESULT: RED sanitizer fault — SUMMARY: AddressSanitizer: 1272 byte(s) leaked (tests may have # all passed; the coverage build aborts at exit on an ASan/LSan fault — often shown only @@ -73,6 +83,15 @@ trap 'rm -f "$LOG" "${MARKER:-}"; [[ -n ${PROGRESS_PID:-} ]] && kill "$PROGRESS_ mapfile -t ALL_SUITES < <(find test -maxdepth 1 -type d -name 'test_*' -printf '%f\n' | sort) EXPECTED_COUNT=${#ALL_SUITES[@]} +# Canonical suite count — the registered total, maintained in test/native-suite-count. +# Update that file whenever a test suite is added or removed. +CANONICAL_COUNT_FILE="test/native-suite-count" +if [[ -f $CANONICAL_COUNT_FILE ]]; then + CANONICAL_COUNT=$(tr -d '[:space:]' <"$CANONICAL_COUNT_FILE") +else + CANONICAL_COUNT="" +fi + # Cached object-count for this env, written after each completed build (in the gitignored build # dir). Used as the progress denominator: accurate for a full rebuild (every object recompiles), # only a rough upper bound for an incremental run. @@ -129,6 +148,9 @@ if ! $QUIET; then echo "Running: $PIO test -e $ENV ${PASSTHRU[*]-} (expecting $EXPECTED_COUNT suites)" fi echo "progress: tail -f $PROGRESS_FILE" >&2 +if [[ ! -t 1 ]] && ! $QUIET; then + echo "hint: stdout is a pipe — build errors appear at the top of output and may be lost; use --quiet to get just the RESULT line" >&2 +fi # Run pio, tee to log. PIPESTATUS[0] is pio's real exit (NOT tee's). if $QUIET; then @@ -227,8 +249,41 @@ if ! grep -qE "$PASS_RE" "$LOG"; then exit 1 fi -# AMBER: everything that ran passed, but (full run only) a canonical suite neither ran NOR was -# explicitly skipped — i.e. it silently went missing. SKIPPED suites are accounted for. +# Canonical-count rating suffix — appended to every verdict line so the result is always +# rated against the registered total, not just the directory count. +# If the two counts diverge (suite added/removed without updating native-suite-count), that +# is itself surfaced as AMBER before we reach any verdict. +canonical_rating() { + if [[ -n $CANONICAL_COUNT ]]; then + echo "[canonical: ${RAN_COUNT}/${CANONICAL_COUNT}]" + fi +} + +# AMBER: directory count disagrees with native-suite-count — file needs updating. +if [[ -n $CANONICAL_COUNT && $EXPECTED_COUNT -ne $CANONICAL_COUNT ]]; then + echo "" + if [[ $EXPECTED_COUNT -gt $CANONICAL_COUNT ]]; then + echo "RESULT: AMBER test/ has $EXPECTED_COUNT suite directories but native-suite-count says $CANONICAL_COUNT — update test/native-suite-count after registering new suites" + else + echo "RESULT: AMBER test/ has $EXPECTED_COUNT suite directories but native-suite-count says $CANONICAL_COUNT — update test/native-suite-count after removing suites" + fi + exit 2 +fi + +# AMBER: individual test cases were skipped (Unity TEST_IGNORE → :IGNORE: in output). +# Applies to both full and filtered runs — a skipped test case is a lost signal either way. +mapfile -t IGNORED_TESTS < <(grep -oE '[^:]+:[0-9]+:[^:]+:IGNORE:.*' "$LOG" 2>/dev/null | sed 's/:IGNORE:.*//' | sort -u) +IGNORED_COUNT=${#IGNORED_TESTS[@]} +if [[ $IGNORED_COUNT -gt 0 ]]; then + IGNORE_DETAIL="$(printf '%s\n' "${IGNORED_TESTS[@]}" | head -5 | sed 's/^/ /')" + echo "" + echo "$IGNORE_DETAIL" + echo "" + echo "RESULT: AMBER ${IGNORED_COUNT} test case(s) ignored $(canonical_rating)" + exit 2 +fi + +# AMBER: full run only — a canonical suite neither ran NOR was explicitly skipped (silently missing). ACCOUNTED_COUNT=$((RAN_COUNT + ${#SKIPPED_SUITES[@]})) if [[ -z $FILTER && $ACCOUNTED_COUNT -lt $EXPECTED_COUNT ]]; then missing=() @@ -236,14 +291,21 @@ if [[ -z $FILTER && $ACCOUNTED_COUNT -lt $EXPECTED_COUNT ]]; then printf '%s\n' "${RAN_SUITES[@]}" "${SKIPPED_SUITES[@]}" | grep -qx "$s" || missing+=("$s") done echo "" - echo "RESULT: AMBER ${RAN_COUNT}/${EXPECTED_COUNT} suites ran (missing: ${missing[*]}) — all that ran passed" + echo "RESULT: AMBER ${RAN_COUNT}/${EXPECTED_COUNT} suites ran (missing: ${missing[*]}) — all that ran passed $(canonical_rating)" exit 2 fi -# GREEN. +# FILTERED: a -f run completed cleanly. Suites outside the filter were intentionally not run; +# this is not a quality signal and is distinct from suites that went missing unexpectedly. if [[ -n $FILTER ]]; then - echo "RESULT: GREEN ${RAN_COUNT} suite(s) passed (filtered: $FILTER)" -else - echo "RESULT: GREEN ${RAN_COUNT}/${EXPECTED_COUNT} suites passed" + not_run=() + for s in "${ALL_SUITES[@]}"; do + printf '%s\n' "${RAN_SUITES[@]}" "${SKIPPED_SUITES[@]}" | grep -qx "$s" || not_run+=("$s") + done + echo "RESULT: FILTERED ${RAN_COUNT}/${EXPECTED_COUNT} suites ran (not run: ${not_run[*]}) — filtered: $FILTER $(canonical_rating)" + exit 3 fi + +# GREEN: all canonical suites ran, all passed, no ignored test cases. +echo "RESULT: GREEN ${RAN_COUNT}/${EXPECTED_COUNT} suites passed $(canonical_rating)" exit 0 diff --git a/test/native-suite-count b/test/native-suite-count new file mode 100644 index 000000000..a45fd52cc --- /dev/null +++ b/test/native-suite-count @@ -0,0 +1 @@ +24 From 5a81fb5c695f05dff52a0b713ee518ee818e5b0d Mon Sep 17 00:00:00 2001 From: nomdetom Date: Thu, 25 Jun 2026 13:42:31 +0100 Subject: [PATCH 38/86] fix lint --- .github/copilot-instructions.md | 26 ++++++++++++++------------ AGENTS.md | 26 ++++++++++++++------------ CLAUDE.md | 26 ++++++++++++++------------ 3 files changed, 42 insertions(+), 36 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index d135e46fc..6639407eb 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -1,20 +1,22 @@ # Meshtastic Firmware - Copilot Instructions > **TL;DR** -> | | | -> |---|---| -> | Local tests | `./bin/run-tests.sh` (exit 0 GREEN · 1 RED · 2 AMBER · 3 FILTERED) | -> | Hardware tests | `./mcp-server/run-tests.sh` | -> | Format | `trunk fmt` | -> | Mirror docs | `AGENTS.md` (short pointer for agents that don't read this file) · `CLAUDE.md` (Claude Code) | +> +> | | | +> | -------------- | -------------------------------------------------------------------------------------------- | +> | Local tests | `./bin/run-tests.sh` (exit 0 GREEN · 1 RED · 2 AMBER · 3 FILTERED) | +> | Hardware tests | `./mcp-server/run-tests.sh` | +> | Format | `trunk fmt` | +> | Mirror docs | `AGENTS.md` (short pointer for agents that don't read this file) · `CLAUDE.md` (Claude Code) | > > **Need this? It's here.** -> | | | -> |---|---| -> | General helpers (clamp, UTF-8, string fmt…) | `src/meshUtils.h` | -> | Logging macros (LOG_DEBUG / INFO / WARN…) | `src/DebugConfiguration.h` | -> | New module skeleton | inherit `ProtobufModule` in `src/mesh/ProtobufModule.h` | -> | Observer / event wiring | `src/Observer.h` | +> +> | | | +> | ------------------------------------------- | ---------------------------------------------------------- | +> | General helpers (clamp, UTF-8, string fmt…) | `src/meshUtils.h` | +> | Logging macros (LOG_DEBUG / INFO / WARN…) | `src/DebugConfiguration.h` | +> | New module skeleton | inherit `ProtobufModule` in `src/mesh/ProtobufModule.h` | +> | Observer / event wiring | `src/Observer.h` | This document provides context and guidelines for AI assistants working with the Meshtastic firmware codebase. diff --git a/AGENTS.md b/AGENTS.md index 2124f65f4..0a0c82406 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,20 +1,22 @@ # Agent instructions > **TL;DR** -> | | | -> |---|---| -> | Local tests | `./bin/run-tests.sh` (exit 0 GREEN · 1 RED · 2 AMBER · 3 FILTERED) | -> | Hardware tests | `./mcp-server/run-tests.sh` | -> | Format | `trunk fmt` | -> | Mirror docs | `.github/copilot-instructions.md` (canonical) · `CLAUDE.md` (Claude Code) | +> +> | | | +> | -------------- | ------------------------------------------------------------------------- | +> | Local tests | `./bin/run-tests.sh` (exit 0 GREEN · 1 RED · 2 AMBER · 3 FILTERED) | +> | Hardware tests | `./mcp-server/run-tests.sh` | +> | Format | `trunk fmt` | +> | Mirror docs | `.github/copilot-instructions.md` (canonical) · `CLAUDE.md` (Claude Code) | > > **Need this? It's here.** -> | | | -> |---|---| -> | General helpers (clamp, UTF-8, string fmt…) | `src/meshUtils.h` | -> | Logging macros (LOG_DEBUG / INFO / WARN…) | `src/DebugConfiguration.h` | -> | New module skeleton | inherit `ProtobufModule` in `src/mesh/ProtobufModule.h` | -> | Observer / event wiring | `src/Observer.h` | +> +> | | | +> | ------------------------------------------- | ---------------------------------------------------------- | +> | General helpers (clamp, UTF-8, string fmt…) | `src/meshUtils.h` | +> | Logging macros (LOG_DEBUG / INFO / WARN…) | `src/DebugConfiguration.h` | +> | New module skeleton | inherit `ProtobufModule` in `src/mesh/ProtobufModule.h` | +> | Observer / event wiring | `src/Observer.h` | This repository is the [Meshtastic](https://meshtastic.org) firmware — a C++17 embedded codebase targeting ESP32 / nRF52 / RP2040 / STM32WL / Linux-Portduino LoRa mesh radios — plus a Python MCP server in `mcp-server/` that AI agents use to flash, configure, and test connected devices. diff --git a/CLAUDE.md b/CLAUDE.md index 3b24bf807..f4cac6e16 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,20 +1,22 @@ # Claude Code instructions > **TL;DR** -> | | | -> |---|---| -> | Local tests | `./bin/run-tests.sh` (exit 0 GREEN · 1 RED · 2 AMBER · 3 FILTERED) | -> | Hardware tests | `./mcp-server/run-tests.sh` | -> | Format | `trunk fmt` | -> | Mirror docs | `.github/copilot-instructions.md` (canonical) · `AGENTS.md` | +> +> | | | +> | -------------- | ------------------------------------------------------------------ | +> | Local tests | `./bin/run-tests.sh` (exit 0 GREEN · 1 RED · 2 AMBER · 3 FILTERED) | +> | Hardware tests | `./mcp-server/run-tests.sh` | +> | Format | `trunk fmt` | +> | Mirror docs | `.github/copilot-instructions.md` (canonical) · `AGENTS.md` | > > **Need this? It's here.** -> | | | -> |---|---| -> | General helpers (clamp, UTF-8, string fmt…) | `src/meshUtils.h` | -> | Logging macros (LOG_DEBUG / INFO / WARN…) | `src/DebugConfiguration.h` | -> | New module skeleton | inherit `ProtobufModule` in `src/mesh/ProtobufModule.h` | -> | Observer / event wiring | `src/Observer.h` | +> +> | | | +> | ------------------------------------------- | ---------------------------------------------------------- | +> | General helpers (clamp, UTF-8, string fmt…) | `src/meshUtils.h` | +> | Logging macros (LOG_DEBUG / INFO / WARN…) | `src/DebugConfiguration.h` | +> | New module skeleton | inherit `ProtobufModule` in `src/mesh/ProtobufModule.h` | +> | Observer / event wiring | `src/Observer.h` | **Read `.github/copilot-instructions.md` first.** That file is the canonical agent-facing document for this repo. It covers project layout, coding conventions, the build system, CI/CD, the native C++ test suite, and the MCP Server & Hardware Test Harness. Read it top-to-bottom before starting any non-trivial change. From f1b1e35a79d4966f9b593ad90dc18b3a06f74483 Mon Sep 17 00:00:00 2001 From: Clive Blackledge Date: Thu, 25 Jun 2026 13:57:27 -0700 Subject: [PATCH 39/86] Skip Bluetooth wait when Bluetooth is disabled (#10571) Make the PowerFSM DARK-to-LS timeout immediate when Bluetooth support is compiled out or config.bluetooth.enabled is false. The configured wait_bluetooth_secs default behavior is unchanged when Bluetooth is enabled. --- src/PowerFSM.cpp | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/PowerFSM.cpp b/src/PowerFSM.cpp index 0a1229f16..ffd2475fb 100644 --- a/src/PowerFSM.cpp +++ b/src/PowerFSM.cpp @@ -58,6 +58,23 @@ static bool isPowered() return !isPowerSavingMode && powerStatus && (!powerStatus->getHasBattery() || powerStatus->getHasUSB()); } +static bool isBluetoothEnabledForPowerFSM() +{ +#if HAS_BLUETOOTH && !MESHTASTIC_EXCLUDE_BLUETOOTH + return config.bluetooth.enabled; +#else + return false; +#endif +} + +static uint32_t getBluetoothWaitMs() +{ + if (!isBluetoothEnabledForPowerFSM()) + return 0; + + return Default::getConfiguredOrDefaultMs(config.power.wait_bluetooth_secs, default_wait_bluetooth_secs); +} + #if defined(T5_S3_EPAPER_PRO) static void t5BacklightOffForSleep() { @@ -429,10 +446,7 @@ void PowerFSM_setup() // If ESP32 and using power-saving, timer mover from DARK to light-sleep // Also serves purpose of the old DARK to DARK transition(?) See https://github.com/meshtastic/firmware/issues/3517 - powerFSM.add_timed_transition( - &stateDARK, &stateLS, - Default::getConfiguredOrDefaultMs(config.power.wait_bluetooth_secs, default_wait_bluetooth_secs), NULL, - "Bluetooth timeout"); + powerFSM.add_timed_transition(&stateDARK, &stateLS, getBluetoothWaitMs(), NULL, "Bluetooth timeout"); } else { // If ESP32, but not using power-saving, check periodically if config has drifted out of stateDark powerFSM.add_timed_transition(&stateDARK, &stateDARK, From 98fa4a67db9b97f5ad37473ca617fe016545b608 Mon Sep 17 00:00:00 2001 From: Clive Blackledge Date: Thu, 25 Jun 2026 15:05:27 -0700 Subject: [PATCH 40/86] [feature] Free a little memory used by bluetooth if wifi is enabled or bluetooth is disabled (#10398) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * esp32: release BTDM heap when Bluetooth inactive Release ESP-IDF BTDM memory after config load when Bluetooth is disabled or WiFi is enabled, recovering heap on ESP32 targets where BLE won’t be used for this boot. * Address BT memory release review comments --------- Co-authored-by: Ben Meadors --- src/main.cpp | 5 +++ src/main.h | 5 ++- src/modules/AdminModule.cpp | 1 + src/platform/esp32/main-esp32.cpp | 72 +++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 1 deletion(-) diff --git a/src/main.cpp b/src/main.cpp index 1ca52c059..333db08cd 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -792,6 +792,11 @@ void setup() // We do this as early as possible because this loads preferences from flash // but we need to do this after main cpu init (esp32setup), because we need the random seed set nodeDB = new NodeDB; +#ifdef ARCH_ESP32 + // Config is loaded now, and Bluetooth has not been initialized yet. If the + // saved config will keep Bluetooth inactive, return its reserved memory early. + esp32ReleaseBluetoothMemoryIfUnused(); +#endif // Initialize transmit history to persist broadcast throttle timers across reboots TransmitHistory::getInstance()->loadFromDisk(); diff --git a/src/main.h b/src/main.h index 661fdd9fb..345869129 100644 --- a/src/main.h +++ b/src/main.h @@ -11,7 +11,7 @@ #include "mesh/generated/meshtastic/telemetry.pb.h" #include #include -#if defined(ARCH_ESP32) && !defined(CONFIG_IDF_TARGET_ESP32S2) +#if defined(ARCH_ESP32) && !defined(CONFIG_IDF_TARGET_ESP32S2) && !MESHTASTIC_EXCLUDE_BLUETOOTH #include "nimble/NimbleBluetooth.h" extern NimbleBluetooth *nimbleBluetooth; #endif @@ -114,6 +114,9 @@ extern bool runASAP; extern bool pauseBluetoothLogging; void nrf52Setup(), esp32Setup(), nrf52Loop(), esp32Loop(), rp2040Setup(), clearBonds(), enterDfuMode(); +#ifdef ARCH_ESP32 +void esp32ReleaseBluetoothMemoryIfUnused(); +#endif meshtastic_DeviceMetadata getDeviceMetadata(); #if !MESHTASTIC_EXCLUDE_I2C diff --git a/src/modules/AdminModule.cpp b/src/modules/AdminModule.cpp index 8bcfcd878..39ae6b788 100644 --- a/src/modules/AdminModule.cpp +++ b/src/modules/AdminModule.cpp @@ -1753,6 +1753,7 @@ void disableBluetooth() #ifdef ARCH_ESP32 if (nimbleBluetooth) nimbleBluetooth->deinit(); + esp32ReleaseBluetoothMemoryIfUnused(); #elif defined(ARCH_NRF52) if (nrf52Bluetooth) nrf52Bluetooth->shutdown(); diff --git a/src/platform/esp32/main-esp32.cpp b/src/platform/esp32/main-esp32.cpp index bcd618c7e..2e0313ad0 100644 --- a/src/platform/esp32/main-esp32.cpp +++ b/src/platform/esp32/main-esp32.cpp @@ -8,6 +8,11 @@ #include "nimble/NimbleBluetooth.h" #endif +#if !defined(CONFIG_IDF_TARGET_ESP32S2) && !MESHTASTIC_EXCLUDE_BLUETOOTH && __has_include() +#include +#define CAN_RELEASE_BT_MEMORY 1 +#endif + #include #if HAS_WIFI @@ -30,9 +35,53 @@ void variant_shutdown() __attribute__((weak)); void variant_shutdown() {} +#if !defined(CONFIG_IDF_TARGET_ESP32S2) && !MESHTASTIC_EXCLUDE_BLUETOOTH +static bool bluetoothMemoryReleased; +static bool bluetoothMemoryReleaseWarned; +#endif + +#if !defined(CONFIG_IDF_TARGET_ESP32S2) && !MESHTASTIC_EXCLUDE_BLUETOOTH +static bool isNetworkConfiguredToDisableBluetooth() +{ +#if HAS_WIFI + return isWifiAvailable(); +#elif defined(USE_WS5500) || defined(USE_CH390D) + return config.network.wifi_enabled; +#else + return false; +#endif +} + +static bool shouldReleaseBluetoothMemory() +{ + // On ESP32 targets WiFi and BLE share radio resources. When WiFi is configured for this boot, + // BLE will not be started, so its reserved memory can be returned to the heap until reboot. + if (isNetworkConfiguredToDisableBluetooth()) { + return true; + } + return !config.bluetooth.enabled; +} + +static const char *getBluetoothReleaseReason() +{ + if (isNetworkConfiguredToDisableBluetooth()) { + return "WiFi is enabled"; + } + return "Bluetooth is disabled"; +} +#endif + #if !defined(CONFIG_IDF_TARGET_ESP32S2) && !MESHTASTIC_EXCLUDE_BLUETOOTH void setBluetoothEnable(bool enable) { + if (enable && bluetoothMemoryReleased) { + if (!shouldReleaseBluetoothMemory() && !bluetoothMemoryReleaseWarned) { + bluetoothMemoryReleaseWarned = true; + LOG_WARN("Bluetooth memory has been released; reboot to re-enable Bluetooth"); + } + return; + } + #if defined(USE_WS5500) || defined(USE_CH390D) if ((config.bluetooth.enabled == true) && (config.network.wifi_enabled == false)) #elif HAS_WIFI @@ -58,6 +107,29 @@ void setBluetoothEnable(bool enable) {} void updateBatteryLevel(uint8_t level) {} #endif +void esp32ReleaseBluetoothMemoryIfUnused() +{ +#ifdef CAN_RELEASE_BT_MEMORY + if (bluetoothMemoryReleased || !shouldReleaseBluetoothMemory()) { + return; + } + + const int32_t heapBefore = ESP.getHeapSize(); + const int32_t freeBefore = ESP.getFreeHeap(); + + // ESP_BT_MODE_BTDM releases all BT/BLE controller and host memory for this boot. + // It is intentionally irreversible until reboot, matching the runtime config behavior. + esp_err_t err = esp_bt_mem_release(ESP_BT_MODE_BTDM); + if (err == ESP_OK) { + bluetoothMemoryReleased = true; + LOG_INFO("Released BTDM memory because %s: heap %+d, free %+d", getBluetoothReleaseReason(), + (int32_t)ESP.getHeapSize() - heapBefore, (int32_t)ESP.getFreeHeap() - freeBefore); + } else { + LOG_WARN("BTDM memory release failed: %d", err); + } +#endif +} + void getMacAddr(uint8_t *dmac) { #if defined(CONFIG_IDF_TARGET_ESP32C6) && defined(CONFIG_SOC_IEEE802154_SUPPORTED) From 00f5fa80be6531e6501b38b8a3f93c20bb718cda Mon Sep 17 00:00:00 2001 From: Clive Blackledge Date: Thu, 25 Jun 2026 18:48:38 -0700 Subject: [PATCH 41/86] Add native Portduino malloc shim (#10677) * Add native Portduino malloc shim * Taking copilots advice Honestly I don't think it matters that much, but if it makes copilot happy, return ret if something weird happens. Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Ben Meadors Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- variants/native/portduino/malloc.h | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 variants/native/portduino/malloc.h diff --git a/variants/native/portduino/malloc.h b/variants/native/portduino/malloc.h new file mode 100644 index 000000000..3c4d7a25a --- /dev/null +++ b/variants/native/portduino/malloc.h @@ -0,0 +1,21 @@ +#pragma once + +#if defined(__APPLE__) +#include +#include +#include + +// LovyanGFX includes the Linux header name; bridge it for Darwin native builds. +static inline void *memalign(size_t alignment, size_t size) +{ + void *ptr = NULL; + int ret = posix_memalign(&ptr, alignment, size); + if (ret != 0) { + errno = ret; + return NULL; + } + return ptr; +} +#else +#include_next +#endif From ad132e9540c2e7c206fb2174660b64a77f6bbd8a Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Fri, 26 Jun 2026 07:19:33 -0500 Subject: [PATCH 42/86] Protobufs update --- protobufs | 2 +- src/mesh/generated/meshtastic/deviceonly.pb.h | 2 +- src/mesh/generated/meshtastic/localonly.pb.h | 14 +- src/mesh/generated/meshtastic/mesh.pb.cpp | 3 + src/mesh/generated/meshtastic/mesh.pb.h | 74 ++++++++- .../generated/meshtastic/mesh_beacon.pb.cpp | 12 ++ .../generated/meshtastic/mesh_beacon.pb.h | 72 ++++++++ .../generated/meshtastic/module_config.pb.cpp | 10 +- .../generated/meshtastic/module_config.pb.h | 154 +++++++++++++++++- src/mesh/generated/meshtastic/portnums.pb.h | 5 + 10 files changed, 331 insertions(+), 17 deletions(-) create mode 100644 src/mesh/generated/meshtastic/mesh_beacon.pb.cpp create mode 100644 src/mesh/generated/meshtastic/mesh_beacon.pb.h diff --git a/protobufs b/protobufs index 03314e639..f680aace2 160000 --- a/protobufs +++ b/protobufs @@ -1 +1 @@ -Subproject commit 03314e63950bda910695eaeb4ba8169dae0425ef +Subproject commit f680aace29ebd3234291db3d260c72eba39c5a26 diff --git a/src/mesh/generated/meshtastic/deviceonly.pb.h b/src/mesh/generated/meshtastic/deviceonly.pb.h index 5bd116730..c838dc21d 100644 --- a/src/mesh/generated/meshtastic/deviceonly.pb.h +++ b/src/mesh/generated/meshtastic/deviceonly.pb.h @@ -452,7 +452,7 @@ extern const pb_msgdesc_t meshtastic_BackupPreferences_msg; /* Maximum encoded size of messages (where known) */ /* meshtastic_NodeDatabase_size depends on runtime parameters */ #define MESHTASTIC_MESHTASTIC_DEVICEONLY_PB_H_MAX_SIZE meshtastic_BackupPreferences_size -#define meshtastic_BackupPreferences_size 2410 +#define meshtastic_BackupPreferences_size 2738 #define meshtastic_ChannelFile_size 718 #define meshtastic_DeviceState_size 1944 #define meshtastic_NodeEnvironmentEntry_size 170 diff --git a/src/mesh/generated/meshtastic/localonly.pb.h b/src/mesh/generated/meshtastic/localonly.pb.h index 3cb4a5a57..6fcaed306 100644 --- a/src/mesh/generated/meshtastic/localonly.pb.h +++ b/src/mesh/generated/meshtastic/localonly.pb.h @@ -96,6 +96,9 @@ typedef struct _meshtastic_LocalModuleConfig { /* TAK Config */ bool has_tak; meshtastic_ModuleConfig_TAKConfig tak; + /* MeshBeacon Config */ + bool has_mesh_beacon; + meshtastic_ModuleConfig_MeshBeaconConfig mesh_beacon; } meshtastic_LocalModuleConfig; @@ -105,9 +108,9 @@ extern "C" { /* Initializer values for message structs */ #define meshtastic_LocalConfig_init_default {false, meshtastic_Config_DeviceConfig_init_default, false, meshtastic_Config_PositionConfig_init_default, false, meshtastic_Config_PowerConfig_init_default, false, meshtastic_Config_NetworkConfig_init_default, false, meshtastic_Config_DisplayConfig_init_default, false, meshtastic_Config_LoRaConfig_init_default, false, meshtastic_Config_BluetoothConfig_init_default, 0, false, meshtastic_Config_SecurityConfig_init_default} -#define meshtastic_LocalModuleConfig_init_default {false, meshtastic_ModuleConfig_MQTTConfig_init_default, false, meshtastic_ModuleConfig_SerialConfig_init_default, false, meshtastic_ModuleConfig_ExternalNotificationConfig_init_default, false, meshtastic_ModuleConfig_StoreForwardConfig_init_default, false, meshtastic_ModuleConfig_RangeTestConfig_init_default, false, meshtastic_ModuleConfig_TelemetryConfig_init_default, false, meshtastic_ModuleConfig_CannedMessageConfig_init_default, 0, false, meshtastic_ModuleConfig_AudioConfig_init_default, false, meshtastic_ModuleConfig_RemoteHardwareConfig_init_default, false, meshtastic_ModuleConfig_NeighborInfoConfig_init_default, false, meshtastic_ModuleConfig_AmbientLightingConfig_init_default, false, meshtastic_ModuleConfig_DetectionSensorConfig_init_default, false, meshtastic_ModuleConfig_PaxcounterConfig_init_default, false, meshtastic_ModuleConfig_StatusMessageConfig_init_default, false, meshtastic_ModuleConfig_TrafficManagementConfig_init_default, false, meshtastic_ModuleConfig_TAKConfig_init_default} +#define meshtastic_LocalModuleConfig_init_default {false, meshtastic_ModuleConfig_MQTTConfig_init_default, false, meshtastic_ModuleConfig_SerialConfig_init_default, false, meshtastic_ModuleConfig_ExternalNotificationConfig_init_default, false, meshtastic_ModuleConfig_StoreForwardConfig_init_default, false, meshtastic_ModuleConfig_RangeTestConfig_init_default, false, meshtastic_ModuleConfig_TelemetryConfig_init_default, false, meshtastic_ModuleConfig_CannedMessageConfig_init_default, 0, false, meshtastic_ModuleConfig_AudioConfig_init_default, false, meshtastic_ModuleConfig_RemoteHardwareConfig_init_default, false, meshtastic_ModuleConfig_NeighborInfoConfig_init_default, false, meshtastic_ModuleConfig_AmbientLightingConfig_init_default, false, meshtastic_ModuleConfig_DetectionSensorConfig_init_default, false, meshtastic_ModuleConfig_PaxcounterConfig_init_default, false, meshtastic_ModuleConfig_StatusMessageConfig_init_default, false, meshtastic_ModuleConfig_TrafficManagementConfig_init_default, false, meshtastic_ModuleConfig_TAKConfig_init_default, false, meshtastic_ModuleConfig_MeshBeaconConfig_init_default} #define meshtastic_LocalConfig_init_zero {false, meshtastic_Config_DeviceConfig_init_zero, false, meshtastic_Config_PositionConfig_init_zero, false, meshtastic_Config_PowerConfig_init_zero, false, meshtastic_Config_NetworkConfig_init_zero, false, meshtastic_Config_DisplayConfig_init_zero, false, meshtastic_Config_LoRaConfig_init_zero, false, meshtastic_Config_BluetoothConfig_init_zero, 0, false, meshtastic_Config_SecurityConfig_init_zero} -#define meshtastic_LocalModuleConfig_init_zero {false, meshtastic_ModuleConfig_MQTTConfig_init_zero, false, meshtastic_ModuleConfig_SerialConfig_init_zero, false, meshtastic_ModuleConfig_ExternalNotificationConfig_init_zero, false, meshtastic_ModuleConfig_StoreForwardConfig_init_zero, false, meshtastic_ModuleConfig_RangeTestConfig_init_zero, false, meshtastic_ModuleConfig_TelemetryConfig_init_zero, false, meshtastic_ModuleConfig_CannedMessageConfig_init_zero, 0, false, meshtastic_ModuleConfig_AudioConfig_init_zero, false, meshtastic_ModuleConfig_RemoteHardwareConfig_init_zero, false, meshtastic_ModuleConfig_NeighborInfoConfig_init_zero, false, meshtastic_ModuleConfig_AmbientLightingConfig_init_zero, false, meshtastic_ModuleConfig_DetectionSensorConfig_init_zero, false, meshtastic_ModuleConfig_PaxcounterConfig_init_zero, false, meshtastic_ModuleConfig_StatusMessageConfig_init_zero, false, meshtastic_ModuleConfig_TrafficManagementConfig_init_zero, false, meshtastic_ModuleConfig_TAKConfig_init_zero} +#define meshtastic_LocalModuleConfig_init_zero {false, meshtastic_ModuleConfig_MQTTConfig_init_zero, false, meshtastic_ModuleConfig_SerialConfig_init_zero, false, meshtastic_ModuleConfig_ExternalNotificationConfig_init_zero, false, meshtastic_ModuleConfig_StoreForwardConfig_init_zero, false, meshtastic_ModuleConfig_RangeTestConfig_init_zero, false, meshtastic_ModuleConfig_TelemetryConfig_init_zero, false, meshtastic_ModuleConfig_CannedMessageConfig_init_zero, 0, false, meshtastic_ModuleConfig_AudioConfig_init_zero, false, meshtastic_ModuleConfig_RemoteHardwareConfig_init_zero, false, meshtastic_ModuleConfig_NeighborInfoConfig_init_zero, false, meshtastic_ModuleConfig_AmbientLightingConfig_init_zero, false, meshtastic_ModuleConfig_DetectionSensorConfig_init_zero, false, meshtastic_ModuleConfig_PaxcounterConfig_init_zero, false, meshtastic_ModuleConfig_StatusMessageConfig_init_zero, false, meshtastic_ModuleConfig_TrafficManagementConfig_init_zero, false, meshtastic_ModuleConfig_TAKConfig_init_zero, false, meshtastic_ModuleConfig_MeshBeaconConfig_init_zero} /* Field tags (for use in manual encoding/decoding) */ #define meshtastic_LocalConfig_device_tag 1 @@ -136,6 +139,7 @@ extern "C" { #define meshtastic_LocalModuleConfig_statusmessage_tag 15 #define meshtastic_LocalModuleConfig_traffic_management_tag 16 #define meshtastic_LocalModuleConfig_tak_tag 17 +#define meshtastic_LocalModuleConfig_mesh_beacon_tag 18 /* Struct field encoding specification for nanopb */ #define meshtastic_LocalConfig_FIELDLIST(X, a) \ @@ -176,7 +180,8 @@ X(a, STATIC, OPTIONAL, MESSAGE, detection_sensor, 13) \ X(a, STATIC, OPTIONAL, MESSAGE, paxcounter, 14) \ X(a, STATIC, OPTIONAL, MESSAGE, statusmessage, 15) \ X(a, STATIC, OPTIONAL, MESSAGE, traffic_management, 16) \ -X(a, STATIC, OPTIONAL, MESSAGE, tak, 17) +X(a, STATIC, OPTIONAL, MESSAGE, tak, 17) \ +X(a, STATIC, OPTIONAL, MESSAGE, mesh_beacon, 18) #define meshtastic_LocalModuleConfig_CALLBACK NULL #define meshtastic_LocalModuleConfig_DEFAULT NULL #define meshtastic_LocalModuleConfig_mqtt_MSGTYPE meshtastic_ModuleConfig_MQTTConfig @@ -195,6 +200,7 @@ X(a, STATIC, OPTIONAL, MESSAGE, tak, 17) #define meshtastic_LocalModuleConfig_statusmessage_MSGTYPE meshtastic_ModuleConfig_StatusMessageConfig #define meshtastic_LocalModuleConfig_traffic_management_MSGTYPE meshtastic_ModuleConfig_TrafficManagementConfig #define meshtastic_LocalModuleConfig_tak_MSGTYPE meshtastic_ModuleConfig_TAKConfig +#define meshtastic_LocalModuleConfig_mesh_beacon_MSGTYPE meshtastic_ModuleConfig_MeshBeaconConfig extern const pb_msgdesc_t meshtastic_LocalConfig_msg; extern const pb_msgdesc_t meshtastic_LocalModuleConfig_msg; @@ -206,7 +212,7 @@ extern const pb_msgdesc_t meshtastic_LocalModuleConfig_msg; /* Maximum encoded size of messages (where known) */ #define MESHTASTIC_MESHTASTIC_LOCALONLY_PB_H_MAX_SIZE meshtastic_LocalModuleConfig_size #define meshtastic_LocalConfig_size 757 -#define meshtastic_LocalModuleConfig_size 798 +#define meshtastic_LocalModuleConfig_size 1126 #ifdef __cplusplus } /* extern "C" */ diff --git a/src/mesh/generated/meshtastic/mesh.pb.cpp b/src/mesh/generated/meshtastic/mesh.pb.cpp index ac991c580..4cf8e980c 100644 --- a/src/mesh/generated/meshtastic/mesh.pb.cpp +++ b/src/mesh/generated/meshtastic/mesh.pb.cpp @@ -30,6 +30,9 @@ PB_BIND(meshtastic_StoreForwardPlusPlus, meshtastic_StoreForwardPlusPlus, 2) PB_BIND(meshtastic_RemoteShell, meshtastic_RemoteShell, AUTO) +PB_BIND(meshtastic_BoundingBox, meshtastic_BoundingBox, AUTO) + + PB_BIND(meshtastic_Waypoint, meshtastic_Waypoint, AUTO) diff --git a/src/mesh/generated/meshtastic/mesh.pb.h b/src/mesh/generated/meshtastic/mesh.pb.h index ce92d81f1..78107f116 100644 --- a/src/mesh/generated/meshtastic/mesh.pb.h +++ b/src/mesh/generated/meshtastic/mesh.pb.h @@ -78,10 +78,10 @@ typedef enum _meshtastic_HardwareModel { meshtastic_HardwareModel_STATION_G1 = 25, /* RAK11310 (RP2040 + SX1262) */ meshtastic_HardwareModel_RAK11310 = 26, - /* Makerfabs SenseLoRA Receiver (RP2040 + RFM96) */ - meshtastic_HardwareModel_SENSELORA_RP2040 = 27, - /* Makerfabs SenseLoRA Industrial Monitor (ESP32-S3 + RFM96) */ - meshtastic_HardwareModel_SENSELORA_S3 = 28, + /* Makerfabs Tracker Reserved */ + meshtastic_HardwareModel_MAKERFABS_TRACKER = 27, + /* Makerfabs Reserved */ + meshtastic_HardwareModel_MAKERFABS_RESERVED = 28, /* Canary Radio Company - CanaryOne: https://canaryradio.io/products/canaryone */ meshtastic_HardwareModel_CANARYONE = 29, /* Waveshare RP2040 LoRa - https://www.waveshare.com/rp2040-lora.htm */ @@ -948,6 +948,23 @@ typedef struct _meshtastic_RemoteShell { uint32_t last_rx_seq; } meshtastic_RemoteShell; +/* A rectangular, axis-aligned geographic bounding box. + Used to define a rectangular geofence region for a Waypoint. + Fields are ordered west, south, east, north to match the standard bounding box + convention used by GeoJSON and PMTiles (min longitude, min latitude, max longitude, max latitude), + so the box can drive an offline map extract directly. + All coordinates are in degrees scaled by 1e-7 (same convention as Position and Waypoint). */ +typedef struct _meshtastic_BoundingBox { + /* Western edge of the box - minimum longitude (south-west corner) */ + int32_t longitude_west_i; + /* Southern edge of the box - minimum latitude (south-west corner) */ + int32_t latitude_south_i; + /* Eastern edge of the box - maximum longitude (north-east corner) */ + int32_t longitude_east_i; + /* Northern edge of the box - maximum latitude (north-east corner) */ + int32_t latitude_north_i; +} meshtastic_BoundingBox; + /* Waypoint message, used to share arbitrary locations across the mesh */ typedef struct _meshtastic_Waypoint { /* Id of the waypoint */ @@ -969,6 +986,20 @@ typedef struct _meshtastic_Waypoint { char description[100]; /* Designator icon for the waypoint in the form of a unicode emoji */ uint32_t icon; + /* If greater than zero, defines a circular geofence centred on this waypoint's + location (latitude_i / longitude_i) with this radius in meters. + Zero means the waypoint has no circular geofence. */ + uint32_t geofence_radius; + /* Optional rectangular geofence region for this waypoint. + May be used instead of, or in addition to, geofence_radius. */ + bool has_bounding_box; + meshtastic_BoundingBox bounding_box; + /* If true, a notification should be raised when a tracked node enters this + waypoint's geofence (the circular radius and/or the bounding box). */ + bool notify_on_enter; + /* If true, a notification should be raised when a tracked node exits this + waypoint's geofence (the circular radius and/or the bounding box). */ + bool notify_on_exit; } meshtastic_Waypoint; /* Message for node status */ @@ -1628,6 +1659,7 @@ extern "C" { + #define meshtastic_MeshPacket_priority_ENUMTYPE meshtastic_MeshPacket_Priority #define meshtastic_MeshPacket_delayed_ENUMTYPE meshtastic_MeshPacket_Delayed #define meshtastic_MeshPacket_transport_mechanism_ENUMTYPE meshtastic_MeshPacket_TransportMechanism @@ -1678,7 +1710,8 @@ extern "C" { #define meshtastic_KeyVerification_init_default {0, {0, {0}}, {0, {0}}} #define meshtastic_StoreForwardPlusPlus_init_default {_meshtastic_StoreForwardPlusPlus_SFPP_message_type_MIN, {0, {0}}, {0, {0}}, {0, {0}}, {0, {0}}, 0, 0, 0, 0, 0} #define meshtastic_RemoteShell_init_default {_meshtastic_RemoteShell_OpCode_MIN, 0, 0, 0, {0, {0}}, 0, 0, 0, 0, 0} -#define meshtastic_Waypoint_init_default {0, false, 0, false, 0, 0, 0, "", "", 0} +#define meshtastic_BoundingBox_init_default {0, 0, 0, 0} +#define meshtastic_Waypoint_init_default {0, false, 0, false, 0, 0, 0, "", "", 0, 0, false, meshtastic_BoundingBox_init_default, 0, 0} #define meshtastic_StatusMessage_init_default {""} #define meshtastic_MqttClientProxyMessage_init_default {"", 0, {{0, {0}}}, 0} #define meshtastic_MeshPacket_init_default {0, 0, 0, 0, {meshtastic_Data_init_default}, 0, 0, 0, 0, 0, _meshtastic_MeshPacket_Priority_MIN, 0, _meshtastic_MeshPacket_Delayed_MIN, 0, 0, {0, {0}}, 0, 0, 0, 0, _meshtastic_MeshPacket_TransportMechanism_MIN, 0} @@ -1716,7 +1749,8 @@ extern "C" { #define meshtastic_KeyVerification_init_zero {0, {0, {0}}, {0, {0}}} #define meshtastic_StoreForwardPlusPlus_init_zero {_meshtastic_StoreForwardPlusPlus_SFPP_message_type_MIN, {0, {0}}, {0, {0}}, {0, {0}}, {0, {0}}, 0, 0, 0, 0, 0} #define meshtastic_RemoteShell_init_zero {_meshtastic_RemoteShell_OpCode_MIN, 0, 0, 0, {0, {0}}, 0, 0, 0, 0, 0} -#define meshtastic_Waypoint_init_zero {0, false, 0, false, 0, 0, 0, "", "", 0} +#define meshtastic_BoundingBox_init_zero {0, 0, 0, 0} +#define meshtastic_Waypoint_init_zero {0, false, 0, false, 0, 0, 0, "", "", 0, 0, false, meshtastic_BoundingBox_init_zero, 0, 0} #define meshtastic_StatusMessage_init_zero {""} #define meshtastic_MqttClientProxyMessage_init_zero {"", 0, {{0, {0}}}, 0} #define meshtastic_MeshPacket_init_zero {0, 0, 0, 0, {meshtastic_Data_init_zero}, 0, 0, 0, 0, 0, _meshtastic_MeshPacket_Priority_MIN, 0, _meshtastic_MeshPacket_Delayed_MIN, 0, 0, {0, {0}}, 0, 0, 0, 0, _meshtastic_MeshPacket_TransportMechanism_MIN, 0} @@ -1820,6 +1854,10 @@ extern "C" { #define meshtastic_RemoteShell_flags_tag 8 #define meshtastic_RemoteShell_last_tx_seq_tag 9 #define meshtastic_RemoteShell_last_rx_seq_tag 10 +#define meshtastic_BoundingBox_longitude_west_i_tag 1 +#define meshtastic_BoundingBox_latitude_south_i_tag 2 +#define meshtastic_BoundingBox_longitude_east_i_tag 3 +#define meshtastic_BoundingBox_latitude_north_i_tag 4 #define meshtastic_Waypoint_id_tag 1 #define meshtastic_Waypoint_latitude_i_tag 2 #define meshtastic_Waypoint_longitude_i_tag 3 @@ -1828,6 +1866,10 @@ extern "C" { #define meshtastic_Waypoint_name_tag 6 #define meshtastic_Waypoint_description_tag 7 #define meshtastic_Waypoint_icon_tag 8 +#define meshtastic_Waypoint_geofence_radius_tag 9 +#define meshtastic_Waypoint_bounding_box_tag 10 +#define meshtastic_Waypoint_notify_on_enter_tag 11 +#define meshtastic_Waypoint_notify_on_exit_tag 12 #define meshtastic_StatusMessage_status_tag 1 #define meshtastic_MqttClientProxyMessage_topic_tag 1 #define meshtastic_MqttClientProxyMessage_data_tag 2 @@ -2083,6 +2125,14 @@ X(a, STATIC, SINGULAR, UINT32, last_rx_seq, 10) #define meshtastic_RemoteShell_CALLBACK NULL #define meshtastic_RemoteShell_DEFAULT NULL +#define meshtastic_BoundingBox_FIELDLIST(X, a) \ +X(a, STATIC, SINGULAR, SFIXED32, longitude_west_i, 1) \ +X(a, STATIC, SINGULAR, SFIXED32, latitude_south_i, 2) \ +X(a, STATIC, SINGULAR, SFIXED32, longitude_east_i, 3) \ +X(a, STATIC, SINGULAR, SFIXED32, latitude_north_i, 4) +#define meshtastic_BoundingBox_CALLBACK NULL +#define meshtastic_BoundingBox_DEFAULT NULL + #define meshtastic_Waypoint_FIELDLIST(X, a) \ X(a, STATIC, SINGULAR, UINT32, id, 1) \ X(a, STATIC, OPTIONAL, SFIXED32, latitude_i, 2) \ @@ -2091,9 +2141,14 @@ X(a, STATIC, SINGULAR, UINT32, expire, 4) \ X(a, STATIC, SINGULAR, UINT32, locked_to, 5) \ X(a, STATIC, SINGULAR, STRING, name, 6) \ X(a, STATIC, SINGULAR, STRING, description, 7) \ -X(a, STATIC, SINGULAR, FIXED32, icon, 8) +X(a, STATIC, SINGULAR, FIXED32, icon, 8) \ +X(a, STATIC, SINGULAR, UINT32, geofence_radius, 9) \ +X(a, STATIC, OPTIONAL, MESSAGE, bounding_box, 10) \ +X(a, STATIC, SINGULAR, BOOL, notify_on_enter, 11) \ +X(a, STATIC, SINGULAR, BOOL, notify_on_exit, 12) #define meshtastic_Waypoint_CALLBACK NULL #define meshtastic_Waypoint_DEFAULT NULL +#define meshtastic_Waypoint_bounding_box_MSGTYPE meshtastic_BoundingBox #define meshtastic_StatusMessage_FIELDLIST(X, a) \ X(a, STATIC, SINGULAR, STRING, status, 1) @@ -2402,6 +2457,7 @@ extern const pb_msgdesc_t meshtastic_Data_msg; extern const pb_msgdesc_t meshtastic_KeyVerification_msg; extern const pb_msgdesc_t meshtastic_StoreForwardPlusPlus_msg; extern const pb_msgdesc_t meshtastic_RemoteShell_msg; +extern const pb_msgdesc_t meshtastic_BoundingBox_msg; extern const pb_msgdesc_t meshtastic_Waypoint_msg; extern const pb_msgdesc_t meshtastic_StatusMessage_msg; extern const pb_msgdesc_t meshtastic_MqttClientProxyMessage_msg; @@ -2442,6 +2498,7 @@ extern const pb_msgdesc_t meshtastic_ChunkedPayloadResponse_msg; #define meshtastic_KeyVerification_fields &meshtastic_KeyVerification_msg #define meshtastic_StoreForwardPlusPlus_fields &meshtastic_StoreForwardPlusPlus_msg #define meshtastic_RemoteShell_fields &meshtastic_RemoteShell_msg +#define meshtastic_BoundingBox_fields &meshtastic_BoundingBox_msg #define meshtastic_Waypoint_fields &meshtastic_Waypoint_msg #define meshtastic_StatusMessage_fields &meshtastic_StatusMessage_msg #define meshtastic_MqttClientProxyMessage_fields &meshtastic_MqttClientProxyMessage_msg @@ -2477,6 +2534,7 @@ extern const pb_msgdesc_t meshtastic_ChunkedPayloadResponse_msg; /* meshtastic_resend_chunks_size depends on runtime parameters */ /* meshtastic_ChunkedPayloadResponse_size depends on runtime parameters */ #define MESHTASTIC_MESHTASTIC_MESH_PB_H_MAX_SIZE meshtastic_FromRadio_size +#define meshtastic_BoundingBox_size 20 #define meshtastic_ChunkedPayload_size 245 #define meshtastic_ClientNotification_size 482 #define meshtastic_Compressed_size 239 @@ -2512,7 +2570,7 @@ extern const pb_msgdesc_t meshtastic_ChunkedPayloadResponse_msg; #define meshtastic_StoreForwardPlusPlus_size 377 #define meshtastic_ToRadio_size 504 #define meshtastic_User_size 115 -#define meshtastic_Waypoint_size 165 +#define meshtastic_Waypoint_size 197 #ifdef __cplusplus } /* extern "C" */ diff --git a/src/mesh/generated/meshtastic/mesh_beacon.pb.cpp b/src/mesh/generated/meshtastic/mesh_beacon.pb.cpp new file mode 100644 index 000000000..6a079b7a7 --- /dev/null +++ b/src/mesh/generated/meshtastic/mesh_beacon.pb.cpp @@ -0,0 +1,12 @@ +/* Automatically generated nanopb constant definitions */ +/* Generated by nanopb-0.4.9.1 */ + +#include "meshtastic/mesh_beacon.pb.h" +#if PB_PROTO_HEADER_VERSION != 40 +#error Regenerate this file with the current version of nanopb generator. +#endif + +PB_BIND(meshtastic_MeshBeacon, meshtastic_MeshBeacon, AUTO) + + + diff --git a/src/mesh/generated/meshtastic/mesh_beacon.pb.h b/src/mesh/generated/meshtastic/mesh_beacon.pb.h new file mode 100644 index 000000000..028d8269f --- /dev/null +++ b/src/mesh/generated/meshtastic/mesh_beacon.pb.h @@ -0,0 +1,72 @@ +/* Automatically generated nanopb header */ +/* Generated by nanopb-0.4.9.1 */ + +#ifndef PB_MESHTASTIC_MESHTASTIC_MESH_BEACON_PB_H_INCLUDED +#define PB_MESHTASTIC_MESHTASTIC_MESH_BEACON_PB_H_INCLUDED +#include +#include "meshtastic/channel.pb.h" +#include "meshtastic/config.pb.h" + +#if PB_PROTO_HEADER_VERSION != 40 +#error Regenerate this file with the current version of nanopb generator. +#endif + +/* Struct definitions */ +/* Payload for MESH_BEACON_APP packets. + Periodically broadcast by nodes in beacon mode. + Listeners deliver the text message to the local inbox and cache any offered + channel/preset for the client app to act on — the firmware never auto-applies them. */ +typedef struct _meshtastic_MeshBeacon { + /* Human-readable beacon message. Max 100 bytes enforced by firmware on send. */ + char message[101]; + /* Optional channel (name + PSK) being advertised to listening clients. + A client app may offer to switch the user to this channel; firmware never applies it automatically. */ + bool has_offer_channel; + meshtastic_ChannelSettings offer_channel; + /* Optional region being advertised alongside offer_preset. */ + meshtastic_Config_LoRaConfig_RegionCode offer_region; + /* Optional modem preset being advertised. + Combined with offer_region, tells a client "there is a mesh on this preset/region". */ + bool has_offer_preset; + meshtastic_Config_LoRaConfig_ModemPreset offer_preset; +} meshtastic_MeshBeacon; + + +#ifdef __cplusplus +extern "C" { +#endif + +/* Initializer values for message structs */ +#define meshtastic_MeshBeacon_init_default {"", false, meshtastic_ChannelSettings_init_default, _meshtastic_Config_LoRaConfig_RegionCode_MIN, false, _meshtastic_Config_LoRaConfig_ModemPreset_MIN} +#define meshtastic_MeshBeacon_init_zero {"", false, meshtastic_ChannelSettings_init_zero, _meshtastic_Config_LoRaConfig_RegionCode_MIN, false, _meshtastic_Config_LoRaConfig_ModemPreset_MIN} + +/* Field tags (for use in manual encoding/decoding) */ +#define meshtastic_MeshBeacon_message_tag 1 +#define meshtastic_MeshBeacon_offer_channel_tag 2 +#define meshtastic_MeshBeacon_offer_region_tag 3 +#define meshtastic_MeshBeacon_offer_preset_tag 4 + +/* Struct field encoding specification for nanopb */ +#define meshtastic_MeshBeacon_FIELDLIST(X, a) \ +X(a, STATIC, SINGULAR, STRING, message, 1) \ +X(a, STATIC, OPTIONAL, MESSAGE, offer_channel, 2) \ +X(a, STATIC, SINGULAR, UENUM, offer_region, 3) \ +X(a, STATIC, OPTIONAL, UENUM, offer_preset, 4) +#define meshtastic_MeshBeacon_CALLBACK NULL +#define meshtastic_MeshBeacon_DEFAULT NULL +#define meshtastic_MeshBeacon_offer_channel_MSGTYPE meshtastic_ChannelSettings + +extern const pb_msgdesc_t meshtastic_MeshBeacon_msg; + +/* Defines for backwards compatibility with code written before nanopb-0.4.0 */ +#define meshtastic_MeshBeacon_fields &meshtastic_MeshBeacon_msg + +/* Maximum encoded size of messages (where known) */ +#define MESHTASTIC_MESHTASTIC_MESH_BEACON_PB_H_MAX_SIZE meshtastic_MeshBeacon_size +#define meshtastic_MeshBeacon_size 180 + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif diff --git a/src/mesh/generated/meshtastic/module_config.pb.cpp b/src/mesh/generated/meshtastic/module_config.pb.cpp index f2fe5d967..472d0b1e5 100644 --- a/src/mesh/generated/meshtastic/module_config.pb.cpp +++ b/src/mesh/generated/meshtastic/module_config.pb.cpp @@ -6,7 +6,7 @@ #error Regenerate this file with the current version of nanopb generator. #endif -PB_BIND(meshtastic_ModuleConfig, meshtastic_ModuleConfig, AUTO) +PB_BIND(meshtastic_ModuleConfig, meshtastic_ModuleConfig, 2) PB_BIND(meshtastic_ModuleConfig_MQTTConfig, meshtastic_ModuleConfig_MQTTConfig, AUTO) @@ -57,6 +57,12 @@ PB_BIND(meshtastic_ModuleConfig_AmbientLightingConfig, meshtastic_ModuleConfig_A PB_BIND(meshtastic_ModuleConfig_StatusMessageConfig, meshtastic_ModuleConfig_StatusMessageConfig, AUTO) +PB_BIND(meshtastic_ModuleConfig_MeshBeaconConfig, meshtastic_ModuleConfig_MeshBeaconConfig, 2) + + +PB_BIND(meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget, meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget, AUTO) + + PB_BIND(meshtastic_ModuleConfig_TAKConfig, meshtastic_ModuleConfig_TAKConfig, AUTO) @@ -76,3 +82,5 @@ PB_BIND(meshtastic_RemoteHardwarePin, meshtastic_RemoteHardwarePin, AUTO) + + diff --git a/src/mesh/generated/meshtastic/module_config.pb.h b/src/mesh/generated/meshtastic/module_config.pb.h index c7edfa0aa..b04c358fc 100644 --- a/src/mesh/generated/meshtastic/module_config.pb.h +++ b/src/mesh/generated/meshtastic/module_config.pb.h @@ -5,6 +5,8 @@ #define PB_MESHTASTIC_MESHTASTIC_MODULE_CONFIG_PB_H_INCLUDED #include #include "meshtastic/atak.pb.h" +#include "meshtastic/channel.pb.h" +#include "meshtastic/config.pb.h" #if PB_PROTO_HEADER_VERSION != 40 #error Regenerate this file with the current version of nanopb generator. @@ -112,6 +114,23 @@ typedef enum _meshtastic_ModuleConfig_CannedMessageConfig_InputEventChar { meshtastic_ModuleConfig_CannedMessageConfig_InputEventChar_CANCEL = 24 } meshtastic_ModuleConfig_CannedMessageConfig_InputEventChar; +/* Boolean options for the beacon module, packed into the `flags` bitfield below. + OR the FLAG_* values together; a flag is on when its bit is set. */ +typedef enum _meshtastic_ModuleConfig_MeshBeaconConfig_Flags { + /* No options enabled. */ + meshtastic_ModuleConfig_MeshBeaconConfig_Flags_FLAG_NONE = 0, + /* Enable receiving MESH_BEACON_APP packets from other nodes. + The text portion is delivered to the local message inbox. + Offered channel/preset are stored for the client app to act on. */ + meshtastic_ModuleConfig_MeshBeaconConfig_Flags_FLAG_LISTEN_ENABLED = 1, + /* Enable periodically broadcasting MESH_BEACON_APP packets from this node. */ + meshtastic_ModuleConfig_MeshBeaconConfig_Flags_FLAG_BROADCAST_ENABLED = 2, + /* When both text and offer content are present, split the beacon into a separate + MESH_BEACON_APP (offer only) and TEXT_MESSAGE_APP (text only) packet, so firmware + that only decodes TEXT_MESSAGE_APP still receives the human-readable text. */ + meshtastic_ModuleConfig_MeshBeaconConfig_Flags_FLAG_LEGACY_SPLIT = 4 +} meshtastic_ModuleConfig_MeshBeaconConfig_Flags; + /* Struct definitions */ /* Settings for reporting unencrypted information about our node to a map via MQTT */ typedef struct _meshtastic_ModuleConfig_MapReportSettings { @@ -438,6 +457,71 @@ typedef struct _meshtastic_ModuleConfig_StatusMessageConfig { char node_status[80]; } meshtastic_ModuleConfig_StatusMessageConfig; +/* One entry in the multi-target broadcast list. + The broadcaster transmits one beacon copy per entry, each on its own radio settings. */ +typedef struct _meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget { + /* Modem preset to use for this target. + Falls back to the running config preset if unset. */ + bool has_preset; + meshtastic_Config_LoRaConfig_ModemPreset preset; + /* Region to use for this target. UNSET means use the running config region. */ + meshtastic_Config_LoRaConfig_RegionCode region; + /* Index into the device's channel table (0..MAX_NUM_CHANNELS-1) of the channel to + transmit this target's beacon on. The referenced channel must already be configured + on the node (its key is needed to encrypt). If unset, the default channel for the + preset is used. */ + bool has_channel_index; + uint32_t channel_index; +} meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget; + +/* MeshBeacon module config */ +typedef struct _meshtastic_ModuleConfig_MeshBeaconConfig { + /* Bitwise-OR of Flags values (listen / broadcast / legacy-split toggles). */ + uint32_t flags; + /* Optional: node ID to send beacon messages AS. + When set, the `from` field of outgoing beacon packets is set to this node ID, + making beacons appear to originate from that node. + When unset (0), beacons are sent as the local node. + A remote admin can only set this field to their own node ID. */ + uint32_t broadcast_send_as_node; + /* Message to include in each beacon broadcast. Max 100 bytes enforced by firmware. */ + char broadcast_message[101]; + /* Optional channel (name + PSK) to advertise in the MeshBeacon offer_channel field. */ + bool has_broadcast_offer_channel; + meshtastic_ChannelSettings broadcast_offer_channel; + /* Optional region to advertise in the MeshBeacon offer_region field. */ + meshtastic_Config_LoRaConfig_RegionCode broadcast_offer_region; + /* Optional modem preset to advertise in the MeshBeacon offer_preset field. */ + bool has_broadcast_offer_preset; + meshtastic_Config_LoRaConfig_ModemPreset broadcast_offer_preset; + /* Single-target TX channel: channel settings (name + PSK) to send beacons on. + If unset, beacons go out on the primary channel. Used only when broadcast_targets is empty. + NOTE: the single-target path embeds the ChannelSettings inline here, whereas a + broadcast_targets entry references a channel-table slot by channel_index instead — see + BroadcastTarget. The two paths are equal, first-class options; only this representation differs. */ + bool has_broadcast_on_channel; + meshtastic_ChannelSettings broadcast_on_channel; + /* Region to use when sending beacons on broadcast_on_preset. */ + meshtastic_Config_LoRaConfig_RegionCode broadcast_on_region; + /* Modem preset to use when sending beacons. + If different from current config, the radio is temporarily switched for TX. */ + bool has_broadcast_on_preset; + meshtastic_Config_LoRaConfig_ModemPreset broadcast_on_preset; + /* How often to broadcast, in seconds. Min 3600 (1 h), default 3600. */ + uint32_t broadcast_interval_secs; + /* Multi-target broadcast list. + When non-empty the broadcaster transmits one beacon copy per entry in sequence, + each temporarily switching the radio to that entry's preset/region/channel. + When empty, the broadcaster uses the scalar broadcast_on_preset / broadcast_on_region / + broadcast_on_channel fields instead (the single-target path). + Single- and multi-target are equal, first-class options — neither is preferred or + deprecated. They differ only in how the TX channel is named: broadcast_on_channel embeds a + ChannelSettings inline, while a target references an existing channel-table slot by + channel_index (see BroadcastTarget). */ + pb_size_t broadcast_targets_count; + meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget broadcast_targets[4]; +} meshtastic_ModuleConfig_MeshBeaconConfig; + /* TAK team/role configuration */ typedef struct _meshtastic_ModuleConfig_TAKConfig { /* Team color. @@ -505,6 +589,8 @@ typedef struct _meshtastic_ModuleConfig { meshtastic_ModuleConfig_TrafficManagementConfig traffic_management; /* TAK team/role configuration for TAK_TRACKER */ meshtastic_ModuleConfig_TAKConfig tak; + /* MeshBeacon module config */ + meshtastic_ModuleConfig_MeshBeaconConfig mesh_beacon; } payload_variant; } meshtastic_ModuleConfig; @@ -538,6 +624,10 @@ extern "C" { #define _meshtastic_ModuleConfig_CannedMessageConfig_InputEventChar_MAX meshtastic_ModuleConfig_CannedMessageConfig_InputEventChar_BACK #define _meshtastic_ModuleConfig_CannedMessageConfig_InputEventChar_ARRAYSIZE ((meshtastic_ModuleConfig_CannedMessageConfig_InputEventChar)(meshtastic_ModuleConfig_CannedMessageConfig_InputEventChar_BACK+1)) +#define _meshtastic_ModuleConfig_MeshBeaconConfig_Flags_MIN meshtastic_ModuleConfig_MeshBeaconConfig_Flags_FLAG_NONE +#define _meshtastic_ModuleConfig_MeshBeaconConfig_Flags_MAX meshtastic_ModuleConfig_MeshBeaconConfig_Flags_FLAG_LEGACY_SPLIT +#define _meshtastic_ModuleConfig_MeshBeaconConfig_Flags_ARRAYSIZE ((meshtastic_ModuleConfig_MeshBeaconConfig_Flags)(meshtastic_ModuleConfig_MeshBeaconConfig_Flags_FLAG_LEGACY_SPLIT+1)) + @@ -562,6 +652,14 @@ extern "C" { +#define meshtastic_ModuleConfig_MeshBeaconConfig_broadcast_offer_region_ENUMTYPE meshtastic_Config_LoRaConfig_RegionCode +#define meshtastic_ModuleConfig_MeshBeaconConfig_broadcast_offer_preset_ENUMTYPE meshtastic_Config_LoRaConfig_ModemPreset +#define meshtastic_ModuleConfig_MeshBeaconConfig_broadcast_on_region_ENUMTYPE meshtastic_Config_LoRaConfig_RegionCode +#define meshtastic_ModuleConfig_MeshBeaconConfig_broadcast_on_preset_ENUMTYPE meshtastic_Config_LoRaConfig_ModemPreset + +#define meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget_preset_ENUMTYPE meshtastic_Config_LoRaConfig_ModemPreset +#define meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget_region_ENUMTYPE meshtastic_Config_LoRaConfig_RegionCode + #define meshtastic_ModuleConfig_TAKConfig_team_ENUMTYPE meshtastic_Team #define meshtastic_ModuleConfig_TAKConfig_role_ENUMTYPE meshtastic_MemberRole @@ -586,6 +684,8 @@ extern "C" { #define meshtastic_ModuleConfig_CannedMessageConfig_init_default {0, 0, 0, 0, _meshtastic_ModuleConfig_CannedMessageConfig_InputEventChar_MIN, _meshtastic_ModuleConfig_CannedMessageConfig_InputEventChar_MIN, _meshtastic_ModuleConfig_CannedMessageConfig_InputEventChar_MIN, 0, 0, "", 0} #define meshtastic_ModuleConfig_AmbientLightingConfig_init_default {0, 0, 0, 0, 0} #define meshtastic_ModuleConfig_StatusMessageConfig_init_default {""} +#define meshtastic_ModuleConfig_MeshBeaconConfig_init_default {0, 0, "", false, meshtastic_ChannelSettings_init_default, _meshtastic_Config_LoRaConfig_RegionCode_MIN, false, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, false, meshtastic_ChannelSettings_init_default, _meshtastic_Config_LoRaConfig_RegionCode_MIN, false, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, 0, 0, {meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget_init_default, meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget_init_default, meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget_init_default, meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget_init_default}} +#define meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget_init_default {false, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, _meshtastic_Config_LoRaConfig_RegionCode_MIN, false, 0} #define meshtastic_ModuleConfig_TAKConfig_init_default {_meshtastic_Team_MIN, _meshtastic_MemberRole_MIN} #define meshtastic_RemoteHardwarePin_init_default {0, "", _meshtastic_RemoteHardwarePinType_MIN} #define meshtastic_ModuleConfig_init_zero {0, {meshtastic_ModuleConfig_MQTTConfig_init_zero}} @@ -605,6 +705,8 @@ extern "C" { #define meshtastic_ModuleConfig_CannedMessageConfig_init_zero {0, 0, 0, 0, _meshtastic_ModuleConfig_CannedMessageConfig_InputEventChar_MIN, _meshtastic_ModuleConfig_CannedMessageConfig_InputEventChar_MIN, _meshtastic_ModuleConfig_CannedMessageConfig_InputEventChar_MIN, 0, 0, "", 0} #define meshtastic_ModuleConfig_AmbientLightingConfig_init_zero {0, 0, 0, 0, 0} #define meshtastic_ModuleConfig_StatusMessageConfig_init_zero {""} +#define meshtastic_ModuleConfig_MeshBeaconConfig_init_zero {0, 0, "", false, meshtastic_ChannelSettings_init_zero, _meshtastic_Config_LoRaConfig_RegionCode_MIN, false, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, false, meshtastic_ChannelSettings_init_zero, _meshtastic_Config_LoRaConfig_RegionCode_MIN, false, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, 0, 0, {meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget_init_zero, meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget_init_zero, meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget_init_zero, meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget_init_zero}} +#define meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget_init_zero {false, _meshtastic_Config_LoRaConfig_ModemPreset_MIN, _meshtastic_Config_LoRaConfig_RegionCode_MIN, false, 0} #define meshtastic_ModuleConfig_TAKConfig_init_zero {_meshtastic_Team_MIN, _meshtastic_MemberRole_MIN} #define meshtastic_RemoteHardwarePin_init_zero {0, "", _meshtastic_RemoteHardwarePinType_MIN} @@ -715,6 +817,20 @@ extern "C" { #define meshtastic_ModuleConfig_AmbientLightingConfig_green_tag 4 #define meshtastic_ModuleConfig_AmbientLightingConfig_blue_tag 5 #define meshtastic_ModuleConfig_StatusMessageConfig_node_status_tag 1 +#define meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget_preset_tag 1 +#define meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget_region_tag 2 +#define meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget_channel_index_tag 4 +#define meshtastic_ModuleConfig_MeshBeaconConfig_flags_tag 1 +#define meshtastic_ModuleConfig_MeshBeaconConfig_broadcast_send_as_node_tag 3 +#define meshtastic_ModuleConfig_MeshBeaconConfig_broadcast_message_tag 4 +#define meshtastic_ModuleConfig_MeshBeaconConfig_broadcast_offer_channel_tag 5 +#define meshtastic_ModuleConfig_MeshBeaconConfig_broadcast_offer_region_tag 6 +#define meshtastic_ModuleConfig_MeshBeaconConfig_broadcast_offer_preset_tag 7 +#define meshtastic_ModuleConfig_MeshBeaconConfig_broadcast_on_channel_tag 8 +#define meshtastic_ModuleConfig_MeshBeaconConfig_broadcast_on_region_tag 9 +#define meshtastic_ModuleConfig_MeshBeaconConfig_broadcast_on_preset_tag 10 +#define meshtastic_ModuleConfig_MeshBeaconConfig_broadcast_interval_secs_tag 11 +#define meshtastic_ModuleConfig_MeshBeaconConfig_broadcast_targets_tag 13 #define meshtastic_ModuleConfig_TAKConfig_team_tag 1 #define meshtastic_ModuleConfig_TAKConfig_role_tag 2 #define meshtastic_RemoteHardwarePin_gpio_pin_tag 1 @@ -739,6 +855,7 @@ extern "C" { #define meshtastic_ModuleConfig_statusmessage_tag 14 #define meshtastic_ModuleConfig_traffic_management_tag 15 #define meshtastic_ModuleConfig_tak_tag 16 +#define meshtastic_ModuleConfig_mesh_beacon_tag 17 /* Struct field encoding specification for nanopb */ #define meshtastic_ModuleConfig_FIELDLIST(X, a) \ @@ -757,7 +874,8 @@ X(a, STATIC, ONEOF, MESSAGE, (payload_variant,detection_sensor,payload_var X(a, STATIC, ONEOF, MESSAGE, (payload_variant,paxcounter,payload_variant.paxcounter), 13) \ X(a, STATIC, ONEOF, MESSAGE, (payload_variant,statusmessage,payload_variant.statusmessage), 14) \ X(a, STATIC, ONEOF, MESSAGE, (payload_variant,traffic_management,payload_variant.traffic_management), 15) \ -X(a, STATIC, ONEOF, MESSAGE, (payload_variant,tak,payload_variant.tak), 16) +X(a, STATIC, ONEOF, MESSAGE, (payload_variant,tak,payload_variant.tak), 16) \ +X(a, STATIC, ONEOF, MESSAGE, (payload_variant,mesh_beacon,payload_variant.mesh_beacon), 17) #define meshtastic_ModuleConfig_CALLBACK NULL #define meshtastic_ModuleConfig_DEFAULT NULL #define meshtastic_ModuleConfig_payload_variant_mqtt_MSGTYPE meshtastic_ModuleConfig_MQTTConfig @@ -776,6 +894,7 @@ X(a, STATIC, ONEOF, MESSAGE, (payload_variant,tak,payload_variant.tak), 1 #define meshtastic_ModuleConfig_payload_variant_statusmessage_MSGTYPE meshtastic_ModuleConfig_StatusMessageConfig #define meshtastic_ModuleConfig_payload_variant_traffic_management_MSGTYPE meshtastic_ModuleConfig_TrafficManagementConfig #define meshtastic_ModuleConfig_payload_variant_tak_MSGTYPE meshtastic_ModuleConfig_TAKConfig +#define meshtastic_ModuleConfig_payload_variant_mesh_beacon_MSGTYPE meshtastic_ModuleConfig_MeshBeaconConfig #define meshtastic_ModuleConfig_MQTTConfig_FIELDLIST(X, a) \ X(a, STATIC, SINGULAR, BOOL, enabled, 1) \ @@ -952,6 +1071,31 @@ X(a, STATIC, SINGULAR, STRING, node_status, 1) #define meshtastic_ModuleConfig_StatusMessageConfig_CALLBACK NULL #define meshtastic_ModuleConfig_StatusMessageConfig_DEFAULT NULL +#define meshtastic_ModuleConfig_MeshBeaconConfig_FIELDLIST(X, a) \ +X(a, STATIC, SINGULAR, UINT32, flags, 1) \ +X(a, STATIC, SINGULAR, UINT32, broadcast_send_as_node, 3) \ +X(a, STATIC, SINGULAR, STRING, broadcast_message, 4) \ +X(a, STATIC, OPTIONAL, MESSAGE, broadcast_offer_channel, 5) \ +X(a, STATIC, SINGULAR, UENUM, broadcast_offer_region, 6) \ +X(a, STATIC, OPTIONAL, UENUM, broadcast_offer_preset, 7) \ +X(a, STATIC, OPTIONAL, MESSAGE, broadcast_on_channel, 8) \ +X(a, STATIC, SINGULAR, UENUM, broadcast_on_region, 9) \ +X(a, STATIC, OPTIONAL, UENUM, broadcast_on_preset, 10) \ +X(a, STATIC, SINGULAR, UINT32, broadcast_interval_secs, 11) \ +X(a, STATIC, REPEATED, MESSAGE, broadcast_targets, 13) +#define meshtastic_ModuleConfig_MeshBeaconConfig_CALLBACK NULL +#define meshtastic_ModuleConfig_MeshBeaconConfig_DEFAULT NULL +#define meshtastic_ModuleConfig_MeshBeaconConfig_broadcast_offer_channel_MSGTYPE meshtastic_ChannelSettings +#define meshtastic_ModuleConfig_MeshBeaconConfig_broadcast_on_channel_MSGTYPE meshtastic_ChannelSettings +#define meshtastic_ModuleConfig_MeshBeaconConfig_broadcast_targets_MSGTYPE meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget + +#define meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget_FIELDLIST(X, a) \ +X(a, STATIC, OPTIONAL, UENUM, preset, 1) \ +X(a, STATIC, SINGULAR, UENUM, region, 2) \ +X(a, STATIC, OPTIONAL, UINT32, channel_index, 4) +#define meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget_CALLBACK NULL +#define meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget_DEFAULT NULL + #define meshtastic_ModuleConfig_TAKConfig_FIELDLIST(X, a) \ X(a, STATIC, SINGULAR, UENUM, team, 1) \ X(a, STATIC, SINGULAR, UENUM, role, 2) @@ -982,6 +1126,8 @@ extern const pb_msgdesc_t meshtastic_ModuleConfig_TelemetryConfig_msg; extern const pb_msgdesc_t meshtastic_ModuleConfig_CannedMessageConfig_msg; extern const pb_msgdesc_t meshtastic_ModuleConfig_AmbientLightingConfig_msg; extern const pb_msgdesc_t meshtastic_ModuleConfig_StatusMessageConfig_msg; +extern const pb_msgdesc_t meshtastic_ModuleConfig_MeshBeaconConfig_msg; +extern const pb_msgdesc_t meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget_msg; extern const pb_msgdesc_t meshtastic_ModuleConfig_TAKConfig_msg; extern const pb_msgdesc_t meshtastic_RemoteHardwarePin_msg; @@ -1003,6 +1149,8 @@ extern const pb_msgdesc_t meshtastic_RemoteHardwarePin_msg; #define meshtastic_ModuleConfig_CannedMessageConfig_fields &meshtastic_ModuleConfig_CannedMessageConfig_msg #define meshtastic_ModuleConfig_AmbientLightingConfig_fields &meshtastic_ModuleConfig_AmbientLightingConfig_msg #define meshtastic_ModuleConfig_StatusMessageConfig_fields &meshtastic_ModuleConfig_StatusMessageConfig_msg +#define meshtastic_ModuleConfig_MeshBeaconConfig_fields &meshtastic_ModuleConfig_MeshBeaconConfig_msg +#define meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget_fields &meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget_msg #define meshtastic_ModuleConfig_TAKConfig_fields &meshtastic_ModuleConfig_TAKConfig_msg #define meshtastic_RemoteHardwarePin_fields &meshtastic_RemoteHardwarePin_msg @@ -1015,6 +1163,8 @@ extern const pb_msgdesc_t meshtastic_RemoteHardwarePin_msg; #define meshtastic_ModuleConfig_ExternalNotificationConfig_size 42 #define meshtastic_ModuleConfig_MQTTConfig_size 224 #define meshtastic_ModuleConfig_MapReportSettings_size 14 +#define meshtastic_ModuleConfig_MeshBeaconConfig_BroadcastTarget_size 10 +#define meshtastic_ModuleConfig_MeshBeaconConfig_size 324 #define meshtastic_ModuleConfig_NeighborInfoConfig_size 10 #define meshtastic_ModuleConfig_PaxcounterConfig_size 30 #define meshtastic_ModuleConfig_RangeTestConfig_size 12 @@ -1025,7 +1175,7 @@ extern const pb_msgdesc_t meshtastic_RemoteHardwarePin_msg; #define meshtastic_ModuleConfig_TAKConfig_size 4 #define meshtastic_ModuleConfig_TelemetryConfig_size 50 #define meshtastic_ModuleConfig_TrafficManagementConfig_size 30 -#define meshtastic_ModuleConfig_size 227 +#define meshtastic_ModuleConfig_size 328 #define meshtastic_RemoteHardwarePin_size 21 #ifdef __cplusplus diff --git a/src/mesh/generated/meshtastic/portnums.pb.h b/src/mesh/generated/meshtastic/portnums.pb.h index 494ef4a54..2859d7c1f 100644 --- a/src/mesh/generated/meshtastic/portnums.pb.h +++ b/src/mesh/generated/meshtastic/portnums.pb.h @@ -98,6 +98,11 @@ typedef enum _meshtastic_PortNum { This module allows setting an extra string of status for a node. Broadcasts on change and on a timer, possibly once a day. */ meshtastic_PortNum_NODE_STATUS_APP = 36, + /* Beacon module broadcast packets. + ENCODING: protobuf + Periodically broadcast by nodes in beacon mode; received by nodes with MeshBeaconConfig.FLAG_LISTEN_ENABLED. + Carries a text message plus optional channel/preset offers for client apps. */ + meshtastic_PortNum_MESH_BEACON_APP = 37, /* Provides a hardware serial interface to send and receive from the Meshtastic network. Connect to the RX/TX pins of a device with 38400 8N1. Packets received from the Meshtastic network is forwarded to the RX pin while sending a packet to TX will go out to the Mesh network. From ec5d2303052a9cea4164ea32da7e09eb5dddb9c9 Mon Sep 17 00:00:00 2001 From: Tom <116762865+NomDeTom@users.noreply.github.com> Date: Sat, 27 Jun 2026 14:20:51 +0100 Subject: [PATCH 43/86] Feat/mesh beacon (#10618) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Tips robot virtual node / relayer to different LoRa modes & channels Note that this commit has details hardcoded for the Wellington (NZ) mesh, and also requires the following patch to the protobufs: ----- diff --git a/meshtastic/mesh.proto b/meshtastic/mesh.proto index 03162d8..ec54c99 100644 --- a/meshtastic/mesh.proto +++ b/meshtastic/mesh.proto @@ -1393,6 +1393,21 @@ message MeshPacket { * Set by the firmware internally, clients are not supposed to set this. */ uint32 tx_after = 20; + + /* + * The modem preset to use fo rthis packet + */ + uint32 modem_preset = 21; + + /* + * The frequency slot to use for this packet + */ + uint32 frequency_slot = 22; + + /* + * Whether the packet has a nonstandard radio config + */ + bool nonstandard_radio_config = 23; } /* ----- * fix: repair mesh tips CI build * feat: add MeshBeacon module (Phase 1 — proto + generated code + initial stub) * feat(beacon): implement broadcaster + listener (phases 2-5) * feat(beacon): wire RadioLibInterface hooks + admin validation (phases 6-7) * fix(beacon): fix LocalModuleConfig flat access (no payload_variant), add localonly proto field * feat(beacon): fix broadcaster inheritance, add preset/region validation + proto cache - MeshBeaconBroadcastModule now inherits ProtobufModule (alongside private MeshBeaconModule + OSThread), giving it allocDataPacket() and setStartDelay() without extra includes. - Payload cache: rebuildCache() encodes the MeshBeacon protobuf once and stores it in payloadCache[]/payloadCacheSize; sendBeacon() only calls rebuildCache() when payloadCacheDirty==true. AdminModule calls invalidateCache() after saving new config so the next broadcast picks up changes. - Region/preset validation in handleSetModuleConfig (mesh_beacon_tag): broadcast_on_preset is validated against the device's current region via RadioInterface::validateConfigLora(); broadcast_offer_region is validated via RadioInterface::validateConfigRegion(). Invalid values are zeroed with a LOG_WARN before saving. * feat(beacon): add unit tests for MeshBeaconModule and AdminModule configuration validation * remove old meshtips * more validation in NodeDB and AdminModule, and userprefs for baked in goodness * copilot is my gravity * mmmmm... beacon * oops * Enhance unit tests for MeshBeaconModule with detailed validation checks and output formatting * new lines. Why not? * finally * legacy mode activate! * Update protobufs (#17) Co-authored-by: NomDeTom <116762865+NomDeTom@users.noreply.github.com> * better logic, fixed a test * updated for packet signing fixed a test added guards for licensed/ham mode * channel numbers * beacon: encrypt on the beacon channel PSK; fix split note When broadcast_on_channel overrides the primary channel's name/PSK, the beacon was encrypted with the PRIMARY PSK: perhapsEncode keys encryption off the primary slot, but the radio-thread channel switch happens only after encryption. sendBeaconPacket() now installs the beacon channel into the primary slot for the synchronous duration of send() (cooperative threading => no interleaving) so encryption/hash use the beacon channel, then restores it. A shared beaconChannelSettings() helper builds the channel for both the encrypt-time swap and the RF-time swap so the key+hash cannot drift. Also: correct the legacy-split comments (both packets go out on the same beacon radio settings, not the normal config) and merge the two consecutive `if (hasText)` blocks in the listener (cppcheck duplicateCondition). Tests: add channelPskOverride_swapsBeaconChannelAndRestores and noChannelOverride_doesNotSwapPrimary; MockRouter snapshots the primary channel at send() time. Co-Authored-By: Claude Opus 4.8 (1M context) * test/beacon: drain toPhoneQueue in tearDown to fix LSan leak abort The listener delivers received text via MeshService::sendToPhone(), which enqueues the packet into toPhoneQueue and takes ownership. Nothing dequeues it in tests, so the three listener tests carrying message text stranded a MeshPacket each — 1272 bytes / 3 allocations that LeakSanitizer flagged at process exit, aborting the coverage run (surfaced by pio as [ERRORED] / SIGHUP even though all 40 assertions passed). Drain the phone queue in tearDown (getForPhone()/releaseToPool) so the packets return to packetPool. Suite is now GREEN with no sanitizer abort. Co-Authored-By: Claude Opus 4.8 (1M context) * legacy hop override for zero-hoppers * ever more beacons * beacon: comment out broadcast_send_as_node pending further review Functionality preserved in comments with full signing/has_bitfield notes for when it is re-enabled. Proto tag 3 retained on the wire. Co-Authored-By: Claude Sonnet 4.6 * test/beacon: fix fromIsCustomNodeWhenSet now that send-as-node is disabled broadcast_send_as_node is commented out; from is always the local node. Update the test assertion and doc comment to match current behaviour. Co-Authored-By: Claude Sonnet 4.6 * Update protobufs (#21) Co-authored-by: NomDeTom <116762865+NomDeTom@users.noreply.github.com> * flags for beacons * beacon: do more with less — slot-index targets + validation Multi-target beacons embedded a full ChannelSettings in every BroadcastTarget, blowing ModuleConfig past the 512-byte BLE FromRadio budget so the firmware would not compile. Targets now reference an existing channel-table slot by channel_index and the broadcaster resolves it via channels.getByIndex() at TX time. Net effect: the same multi-target capability for a fraction of the bytes — FromRadio 609 -> 510 B, MeshBeaconConfig 596 -> 324 B, AdminMessage 615 -> 511 B. - proto: BroadcastTarget.channel (embedded) -> channel_index (uint32 ref); regen all generated headers (size constants propagate to admin/localonly/deviceonly/mesh). - broadcaster: resolve channel_index from the channel table; an out-of-range or blank slot falls back to the default channel for the target preset rather than borrowing the primary's name/PSK. - AdminModule: validate broadcast_targets entries on write (region/preset sanitised like the single-target fields; channel_index range-checked). - userPrefs: TARGET__CHANNEL_{NAME,NUM,PSK} collapse to a single CHANNEL_INDEX. - docs: two-step (set_channel -> set_module_config) multi-target setup, inline-vs- reference distinction, and single-/multi-target are equal (not "legacy") options. - tests: target validation + channel-index resolution incl. blank-slot fallback (47/47 green on `./bin/run-tests.sh -e native -f test_mesh_beacon`). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01NRAF5csgsMn6p1zEcFL8Qz * throttling after reboot * address copilot review * simplify * fix(beacon): use 0x%08x for node/packet IDs in logs; register test suite The %#08lx log specifiers passed uint32_t (NodeNum/PacketId) to a %lx length modifier — undefined behaviour on 64-bit (native test) targets and non-standard width. Switch to the project-standard 0x%08x. Also bump test/native-suite-count to 25 for the added test_mesh_beacon suite. clod helped too * copilot & clarity clod helped too * refactor(beacon): use auto for the sanitized config copy clod helped too * fix(beacon): guard empty-payload sends; gate has_mesh_beacon on build flag; document ISR_TX pre-switch clod helped too --------- Co-authored-by: Steve Gilberd Co-authored-by: Darafei Praliaskouski Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) --- docs/mesh_beacon_module.md | 454 +++++++++ src/mesh/Channels.h | 10 +- src/mesh/Default.h | 1 + src/mesh/NodeDB.cpp | 149 +++ src/mesh/RadioLibInterface.cpp | 44 +- src/mesh/Router.cpp | 13 + src/modules/AdminModule.cpp | 99 ++ src/modules/AdminModule.h | 4 + src/modules/MeshBeaconModule.cpp | 611 +++++++++++ src/modules/MeshBeaconModule.h | 163 +++ src/modules/Modules.cpp | 7 + test/native-suite-count | 2 +- test/test_mesh_beacon/test_main.cpp | 1457 +++++++++++++++++++++++++++ userPrefs.jsonc | 31 + 14 files changed, 3037 insertions(+), 8 deletions(-) create mode 100644 docs/mesh_beacon_module.md create mode 100644 src/modules/MeshBeaconModule.cpp create mode 100644 src/modules/MeshBeaconModule.h create mode 100644 test/test_mesh_beacon/test_main.cpp diff --git a/docs/mesh_beacon_module.md b/docs/mesh_beacon_module.md new file mode 100644 index 000000000..617dd797e --- /dev/null +++ b/docs/mesh_beacon_module.md @@ -0,0 +1,454 @@ +# Mesh Beacon Module — Function, Settings, and Client Interface Spec + +Status: draft, tracks firmware branch `feat/mesh-beacon`. +Audience: firmware reviewers (Part 1) and client-app developers — Android / Apple / Web / Python (Part 2). + +The Mesh Beacon module lets a node periodically **advertise the existence of a mesh** to +nodes that are not yet on it — broadcasting a short human-readable message plus an optional +"join offer" (a channel, region, and modem preset). It is the mechanism behind invitations +like _"Join us on NarrowSlow"_: a node sitting on one preset/region can shout an invitation +that listeners on other presets/regions can hear and surface to their user. + +The module is deliberately **advisory**. The firmware never auto-joins an advertised +channel or auto-switches preset/region in response to a received beacon — it delivers the +information to the client app and stops there. All "should I act on this?" decisions belong +to the client and, ultimately, the user. + +--- + +## Part 1 — Function and settings choices + +### 1.1 Two roles in one module + +| Role | Class | Active when | What it does | +| --------------- | --------------------------- | ---------------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| **Broadcaster** | `MeshBeaconBroadcastModule` | `FLAG_BROADCAST_ENABLED` set | Periodically transmits `MESH_BEACON_APP` packets on the configured radio settings. | +| **Listener** | `MeshBeaconListenerModule` | `FLAG_LISTEN_ENABLED` set | Receives `MESH_BEACON_APP` packets and caches the offer for the client (the packet itself flows to the client unchanged). | + +The boolean toggles live in a single `flags` bitfield (see [§1.8](#18-settings-reference-moduleconfigmeshbeaconconfig-tag-17)) — broadcasting and +listening can be enabled independently on the same node. The whole module compiles out under the +`MESHTASTIC_EXCLUDE_BEACON` build flag. + +### 1.2 Wire message + +Beacons travel on a dedicated port number: + +```protobuf +MESH_BEACON_APP = 37 // meshtastic/portnums.proto +ENCODING: protobuf (meshtastic.MeshBeacon) +``` + +```protobuf +message MeshBeacon { + string message = 1; // human-readable text, max 100 bytes (buffer 101) + ChannelSettings offer_channel = 2; // optional advertised channel (name + PSK + slot) + Config.LoRaConfig.RegionCode offer_region = 3; // optional advertised region (UNSET = none) + optional Config.LoRaConfig.ModemPreset offer_preset = 4; // optional advertised preset +} +``` + +`.options` size caps (enforced at generation and on send): +`message ≤ 100`, `offer_channel.name ≤ 12`, `offer_channel.psk ≤ 32`. + +The three `offer_*` fields together describe _"there is a reachable mesh on this +region+preset, here is the channel to use."_ Any subset may be present; an empty message with +a populated offer (or vice-versa) is valid. + +### 1.3 Transmission behaviour + +Every outgoing beacon packet is stamped uniformly (`sendBeacon` → `stampPacket`): + +- `to = NODENUM_BROADCAST` +- `from = local node` (see [§1.6](#16-broadcast_send_as_node-currently-disabled) for the disabled spoof path) +- **`hop_limit = 0`** — beacons are **zero-hop**. They are never rebroadcast by the mesh; only + direct RF neighbours hear them. This is the primary spam-control mechanism. (`hop_start` is + normally `0` too, but `FLAG_LEGACY_SPLIT` raises it to `1` for old-firmware compatibility — see + [§1.5](#15-legacy-split-flag_legacy_split).) +- `priority = BACKGROUND`, `want_ack = false`. + +Broadcasting is additionally gated at runtime by: + +- airtime utilisation (`isTxAllowedAirUtil()`), and +- device role — **`CLIENT_HIDDEN` never broadcasts**. + +#### Interval + +`broadcast_interval_secs` controls cadence. The floor is **3600 s (1 hour)** +(`default_mesh_beacon_min_broadcast_interval_secs`); `0` means "use default". Values below the +floor are silently raised, both at config-set time (AdminModule) and at runtime. + +The cadence is **reboot-safe**. Each broadcast's time is persisted to flash via `TransmitHistory` +(keyed by `MESH_BEACON_APP`), and the broadcaster reads it back on boot — so a node that reboots +(or crash-loops) won't re-broadcast until a full interval has elapsed since its last real send, +rather than firing ~30 s after every boot. The timestamp is written **before** the transmit, so a +brown-out during the high-current LoRa TX still counts as "sent." This mirrors `NodeInfoModule` / +`PositionModule`. + +#### Radio switching for TX + +A beacon's whole point is often to reach a mesh on a _different_ preset/region/channel than the +broadcaster currently runs. Before transmitting a beacon tagged with target radio settings, the +module temporarily reconfigures the radio (`reconfigureForBeaconTX`), sends, then restores the +prior config. Per-packet target settings are held in an 8-entry **sidecar table** keyed by packet +ID — chosen so the `MeshPacket` proto carries no extra per-packet radio fields, and normal +(non-beacon) traffic is never touched. + +Two safety guards run before any radio switch (`beaconTxConfigInvalid`): + +1. **An unlicensed node never keys up on a licensed-only (ham) region.** (The reverse — a licensed + node operating in a non-ham region — is allowed. The switch only touches preset/region/channel, + never `owner.is_licensed`.) +2. **The preset must be valid for the target region** (`validateConfigLora`). + + If either fails, the radio is **not** switched and the radio driver **drops** the packet rather + than letting it fall through onto the current config. + +#### Channel encryption on an override channel + +Encryption keys off the **primary** channel slot, and the radio-thread channel switch happens +_after_ encryption. So when a beacon goes out on an override channel (different name/PSK), the +module installs the beacon channel into the primary slot for the synchronous duration of +`send()`, then restores it (`sendBeaconPacket`). This guarantees the packet is encrypted with the +beacon channel's key and stamped with its hash — not the primary's. Meshtastic threading is +cooperative, so there is no preemption between swap and restore. + +### 1.4 Where beacons are sent: single-target and multi-target + +The broadcaster can send to one set of radio settings or to several. **Single- and multi-target +are equal options — neither is preferred and neither is legacy.** Pick whichever matches the +deployment. + +- **Single-target:** the scalar `broadcast_on_preset` / `broadcast_on_region` / + `broadcast_on_channel` fields describe one destination. Used when `broadcast_targets` is empty. +- **Multi-target:** `broadcast_targets` (repeated `BroadcastTarget`) describes several. When + non-empty it takes over from the scalar `broadcast_on_*` fields, and the broadcaster sends **one + beacon copy per entry**. Each `BroadcastTarget` is `{ optional preset, region, optional channel_index }`, + where `channel_index` references a slot in the node's own channel table (the channel must already be + configured locally — its key is needed to encrypt the beacon). Within one cycle, targets that + resolve to the **same** effective preset/region/channel are de-duplicated — only the first is + transmitted — so an accidentally repeated entry costs no extra airtime. + +#### Same-settings vs. other-settings + +Independent of single/multi, each destination can either reuse the node's **own current radio +settings** or specify **different** ones: + +- **Same-settings ("message of the day"):** leave the preset / region / channel unset. They fall + back to the running config, so the beacon goes out on the node's current mesh with **no radio + switch** — a plain periodic broadcast to whoever is already on this preset/region. +- **Other-settings (cross-mesh invite):** set a preset / region / channel that differs from the + running config. The radio is temporarily switched for that copy's TX, then restored (see + [§1.3](#radio-switching-for-tx)). + +Both modes support both styles: a single-target beacon with no `broadcast_on_*` overrides is a +message-of-the-day on the current mesh; a multi-target list can mix one entry on the current +settings with others on different presets/regions. + +### 1.5 Legacy split (`FLAG_LEGACY_SPLIT`) + +This one flag controls **two** independent legacy-compatibility behaviours. Both are about making +beacons usable by firmware that predates this module. + +**(a) Text/offer packet split.** A combined `MESH_BEACON_APP` packet carries both the text and the +offer, but old firmware only decodes `TEXT_MESSAGE_APP` and would never show the text. When +`FLAG_LEGACY_SPLIT` is set **and both text and offer content are present**, the broadcaster +emits **two** packets on the same beacon radio settings instead of one: + +- **Packet A** — `MESH_BEACON_APP` carrying the **offer only** (no text). +- **Packet B** — `TEXT_MESSAGE_APP` carrying the **text only**. + +This is an independent two-packet decision, not an either/or: offer-only and text-only payloads +still go out as a single packet in their respective cases; only the both-present case splits. + +**(b) `hop_start = 1` override.** When `FLAG_LEGACY_SPLIT` is set, **every** beacon packet it sends +(combined, split-A, or split-B; even same-settings ones) is stamped with `hop_start = 1` while +`hop_limit` stays `0`. Pre-2.7.20 firmware drops `hop_start == 0` packets in a pre-decryption check +before it can read the bitfield, so `hop_start = 1` lets those nodes accept the beacon — and it +remains genuinely zero-hop (`hop_limit = 0` still prevents any rebroadcast). + +> **Side effect for clients:** with `hop_start = 1, hop_limit = 0`, receivers compute +> `hops_away = hop_start − hop_limit = 1`, so a legacy-split beacon reads as **1 hop away** even +> though it arrived over direct RF. Without legacy-split it reads as direct (0). Don't treat a +> beacon's `hops_away` as a reliable distance signal. + +### 1.6 `broadcast_send_as_node` (currently disabled) + +The schema reserves `broadcast_send_as_node` (field 3) to send beacons _as_ another node ID. **The +firmware application of this field is currently commented out pending review**, so beacons always +go out as the local node today. The access-control rule is, however, already enforced in +AdminModule and should be treated as canonical: + +> A remote admin may only set `broadcast_send_as_node` to **their own** node ID +> (`mp.from`). Any other value is rejected and reset to the stored value. + +Design note for when it is re-enabled: it is a _node-ID_ spoof only — it rewrites `from` but forges +no signature. Once `from` is not us, the packet is no longer `isFromUs()`, so the router skips +XEdDSA signing and receivers get an unsigned packet attributed to another node. + +### 1.7 Reception behaviour (listener) + +When `FLAG_LISTEN_ENABLED` is **off**, the router drops incoming `MESH_BEACON_APP` packets up front +(`Router::handleReceived`, same pattern as a disabled NeighborInfo module) — so they reach neither +the modules nor the phone. When it is **on**, the packet flows normally and the listener's +`wantPacket` accepts it (`has_mesh_beacon` + `FLAG_LISTEN_ENABLED` + `portnum == MESH_BEACON_APP`). +On a valid beacon (`handleReceivedProtobuf`): + +1. **Offer → cache.** Any offer (`offer_channel` / `offer_region` / `offer_preset`) is stored in + the static `lastReceivedOffer` (sender, channel, region, preset, `received_at`). `received_at` + is `0` if the node has no RTC fix yet — **consumers must not treat `0` as a valid timestamp.** +2. **Never auto-applied.** The firmware does not switch channel/preset/region from a received + offer. Acting on it is the client app's job. +3. The handler returns `CONTINUE` (not `STOP`), so the original `MESH_BEACON_APP` packet **flows to + the client unchanged** through the normal FromRadio path (see Part 2). The client reads the + `message` field directly from that packet — there is no separate copy. + +The firmware deliberately does **not** unwrap a combined beacon's text into a synthesized +`TEXT_MESSAGE_APP`, and does **not** fire `EVENT_RECEIVED_MSG`: a beacon is an advisory broadcast, +not a personal message, so it must not duplicate the text or wake the device from sleep. If a +broadcaster needs non-beacon-aware clients to see the text, it uses `FLAG_LEGACY_SPLIT`, which sends +a real `TEXT_MESSAGE_APP` over RF (see [§1.5](#15-legacy-split-flag_legacy_split)). + +### 1.8 Settings reference (`ModuleConfig.MeshBeaconConfig`, tag 17) + +| # | Field | Type | Meaning / constraints | +| --- | ------------------------- | ------------------------ | ------------------------------------------------------------------------------------------------------ | +| 1 | `flags` | uint32 (bitfield) | Bitwise-OR of `Flags` values (listen / broadcast / legacy-split toggles). See enum below. | +| 3 | `broadcast_send_as_node` | uint32 | Send-as node ID. **Application disabled in firmware.** Remote admin may only set to own node ID. | +| 4 | `broadcast_message` | string | Text in each broadcast. **Hard-capped at 100 bytes.** | +| 5 | `broadcast_offer_channel` | ChannelSettings | Channel advertised in `offer_channel`. | +| 6 | `broadcast_offer_region` | RegionCode | Region advertised in `offer_region`. Must be a known region or it is cleared. | +| 7 | `broadcast_offer_preset` | optional ModemPreset | Preset advertised in `offer_preset`. Validated against offer region (else cleared). | +| 8 | `broadcast_on_channel` | ChannelSettings | Channel to transmit on (single-target). Empty name → preset display name. | +| 9 | `broadcast_on_region` | RegionCode | Region to transmit on (single-target). | +| 10 | `broadcast_on_preset` | optional ModemPreset | Preset to transmit on (single-target). Validated against on-region (else this + `on_channel` cleared). | +| 11 | `broadcast_interval_secs` | uint32 | Cadence. **Min 3600**, default 3600; `0` = default. | +| 13 | `broadcast_targets` | repeated BroadcastTarget | Multi-target list; when non-empty overrides the single-target `broadcast_on_*` fields. | + +> The three boolean toggles were folded into the `flags` bitfield; field tags 2 and 12 are now +> unused (the branch is unreleased, so the old tags are left as gaps rather than reserved). + +**`Flags` enum** (nested in `MeshBeaconConfig`; OR the values into `flags`): + +| Bit value | Name | Meaning | +| --------- | ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| 0 | `FLAG_NONE` | No options enabled. | +| 1 | `FLAG_LISTEN_ENABLED` | Receive beacons; cache the offer. The packet flows to the client, which reads `message` directly. | +| 2 | `FLAG_BROADCAST_ENABLED` | Periodically broadcast beacons from this node. | +| 4 | `FLAG_LEGACY_SPLIT` | Legacy compatibility: (a) split text+offer into separate `TEXT_MESSAGE_APP` + `MESH_BEACON_APP` packets, and (b) stamp `hop_start = 1` on every beacon so pre-2.7.20 firmware accepts it (see [§1.5](#15-legacy-split-flag_legacy_split)). | + +`BroadcastTarget`: `1 preset` (optional, falls back to running config), `2 region` (`UNSET` = running config), `4 channel_index` (optional `uint32`, index into the node's channel table; if unset, the default channel for the preset is used). Tag `3` is an unused gap — it previously held an embedded `ChannelSettings`, dropped to keep `ModuleConfig` within the BLE `FromRadio` size budget. + +--- + +## Part 2 — Client interface specification + +This section is what a client app needs to integrate with the beacon module. Everything goes +through the **standard admin / ToRadio / FromRadio protocol** — there is no bespoke transport. + +### 2.1 Capability detection + +The module is build-flag optional. Treat it as present when the node's `LocalModuleConfig` +contains a `mesh_beacon` sub-message (`LocalModuleConfig.mesh_beacon`, tag 18). If absent, the +firmware was built with `MESHTASTIC_EXCLUDE_BEACON` — hide the beacon UI. + +### 2.2 Reading and writing configuration + +Standard module-config flow — no new admin messages: + +- **Read:** `AdminMessage.get_module_config_request = ModuleConfig.MeshBeaconConfig` (variant 17). + Reply is `get_module_config_response` with the `mesh_beacon` payload. +- **Write:** `AdminMessage.set_module_config { mesh_beacon = … }`. + +The on/off toggles (listen, broadcast, legacy-split) are bits in the `flags` field, not separate +booleans — read/write them with the `MeshBeaconConfig.Flags` values +(`FLAG_LISTEN_ENABLED = 1`, `FLAG_BROADCAST_ENABLED = 2`, `FLAG_LEGACY_SPLIT = 4`). To toggle one +bit, read the current `flags`, set/clear the bit, and write the whole config back. + +The firmware **sanitises on write** — your value may be silently adjusted. Mirror these rules +client-side so the UI doesn't disagree with the device: + +| Rule | Firmware behaviour | +| --------------------------------------------------------------------------- | ------------------------------------------------------------------------------- | +| `broadcast_message` length | Truncated to 100 bytes. | +| `broadcast_interval_secs` | If non-zero and `< 3600`, raised to 3600. | +| `broadcast_on_preset` invalid for `broadcast_on_region` (or current region) | Cleared, **and `broadcast_on_channel` cleared too.** | +| `broadcast_offer_preset` invalid for offer/current region | Cleared. | +| `broadcast_offer_region` not a known region | Cleared to `UNSET`. | +| `broadcast_targets[i].region` not a known region | That entry's region cleared to `UNSET` (TX falls back to running config). | +| `broadcast_targets[i].preset` invalid for that entry's region | That entry's `preset` and `channel_index` cleared. | +| `broadcast_targets[i].channel_index` ≥ `MAX_NUM_CHANNELS` (8) | That entry's `channel_index` cleared (existence is **not** checked — see §2.5). | +| `broadcast_send_as_node` ≠ sender's node ID (remote admin) | Rejected, reset to stored value. | + +Setting beacon config does **not** trigger a reboot (`shouldReboot = false`); changes take effect +on the next broadcast cycle. After a successful write, **re-read** the config to display the +effective (sanitised) values. + +### 2.3 Receiving beacons + +A received beacon reaches the client as a normal `FromRadio.packet` (`MeshPacket`) — the listener +returns `CONTINUE`, so the packet is **not** consumed on-device. The client must: + +1. Subscribe to the FromRadio packet stream as usual. +2. For packets with `decoded.portnum == MESH_BEACON_APP (37)`, decode `decoded.payload` as a + `meshtastic.MeshBeacon`. +3. Read `message`, `offer_channel`, `offer_region`, `offer_preset` (presence-checked). +4. `packet.from` is the **originating beaconer** (the firmware preserves it). + +> **Requires `FLAG_LISTEN_ENABLED` set in `flags`.** With listening disabled the firmware drops +> received `MESH_BEACON_APP` packets in the router — before they reach the phone or any on-device +> handler — the same way it drops a disabled module's packets (e.g. NeighborInfo). The node still +> physically receives the RF, but the client will not see beacons over the FromRadio stream until +> listening is enabled. + +#### Reading the text — no duplication + +For a beacon-aware client the text is **simply the `message` field of the `MESH_BEACON_APP` +packet** you already decode for the offer (step 3 above). One packet, one field — the firmware does +**not** inject a separate `TEXT_MESSAGE_APP` copy, so there is nothing to deduplicate. + +The only time a beacon's text arrives as a separate `TEXT_MESSAGE_APP` is when the broadcaster set +`FLAG_LEGACY_SPLIT`: in that mode the `MESH_BEACON_APP` carries the **offer only** (empty `message`) +and the text is sent as a normal `TEXT_MESSAGE_APP` over RF, so legacy/non-beacon-aware clients can +display it. These two cases are mutually exclusive — a given beacon's text appears exactly once, +either in `MESH_BEACON_APP.message` (combined) or as a `TEXT_MESSAGE_APP` (legacy-split) — so a +client never needs to dedup. Render whichever it receives. + +### 2.4 Acting on an offer (the core client responsibility) + +When a `MESH_BEACON_APP` carries offer content, present it to the user as an **invitation** — +e.g. _"Node ⟨from⟩ invites you to join '⟨offer_channel.name⟩' on ⟨preset⟩/⟨region⟩."_ Then, only on +explicit user confirmation, apply it by writing normal config: + +- `offer_channel` → add/replace a `Channel` (`set_channel`), typically as a secondary channel. +- `offer_region` / `offer_preset` → `set_config { lora = … }` (`use_preset = true`, set + `modem_preset` and `region`). **Note this changes the node's own radio and will drop it off its + current mesh** — make that consequence explicit in the UI. + +**The firmware will never do any of this for the user. No silent auto-apply.** The on-device +`lastReceivedOffer` cache is a firmware-internal convenience and is **not** currently exposed via +an admin message — clients should source offers from the live `MESH_BEACON_APP` packet stream +(§2.3), not expect a "get last offer" RPC. + +#### Offer trust model — read before applying + +- **The advertised PSK is not a secret.** `offer_channel.psk` is a public join token sent in the + clear inside a broadcast; it is a convenience, not a security boundary. An operator who wants a + genuinely private channel must distribute the PSK out-of-band and leave `offer_channel` unset. + Surface offered channels as **public/open** to the user. +- **Validate before applying.** Reject or warn if `offer_preset` is not valid for `offer_region`, + and **never** apply a licensed-only (ham) region for a user who is not a licensed operator — + mirror the firmware's own guard. +- Beacons are **unsigned** when sent as another node (the disabled send-as path), and even normal + beacons assert nothing about the sender's authority. Treat `from` as informational. + +### 2.5 Configuring this node as a broadcaster + +To make a node advertise a mesh, write `MeshBeaconConfig` with `FLAG_BROADCAST_ENABLED` set in +`flags` and at least one of: a non-empty `broadcast_message`, or offer content +(`broadcast_offer_*`). With neither, the broadcaster has nothing to send and stays silent. + +Typical multi-region invite beacon: + +```text +flags = FLAG_BROADCAST_ENABLED | FLAG_LEGACY_SPLIT // broadcast on; split so legacy nodes still see the text +broadcast_message = "Join us on NarrowSlow!" +broadcast_offer_preset = NARROW_SLOW +broadcast_offer_region = EU_N_868 +broadcast_offer_channel = { name: "MyChannel", psk: <32-byte key> } +broadcast_interval_secs = 3600 +// channel_index points at slots in THIS node's channel table — configure those channels first. +broadcast_targets = [ + { preset: LONG_FAST, region: EU_868, channel_index: 0 }, + { preset: NARROW_SLOW, region: EU_N_868, channel_index: 1 }, +] +``` + +The same fields can be baked in at build time via `userPrefs.jsonc` +(`USERPREFS_MESH_BEACON_*`) — see that file for the full list, including +`USERPREFS_MESH_BEACON_TARGET__*` for multi-target entries. + +#### Single-target vs. multi-target — equal options, different channel representation + +Single-target and multi-target are **equal, first-class options**. Neither is preferred, +deprecated, or a "legacy" fallback — pick whichever matches the deployment (a single-target +beacon with no overrides is a plain message-of-the-day; a multi-target list reaches several +preset/region/channel combinations). The broadcaster uses `broadcast_targets` when it is +non-empty and the scalar `broadcast_on_*` fields when it is empty. + +The one **subtle implementation difference** is how each names its TX channel: + +| Path | TX channel is specified by | Channel name/PSK live… | +| ------------- | ------------------------------------------------------- | ----------------------------------------- | +| Single-target | `broadcast_on_channel` — an embedded `ChannelSettings` | …inline in the beacon config | +| Multi-target | `broadcast_targets[i].channel_index` — a `uint32` index | …in the node's channel table (referenced) | + +This asymmetry is deliberate: embedding a full `ChannelSettings` in every one of the (up to +four) targets would push `ModuleConfig` past the BLE `FromRadio` size limit, so a target +references an already-configured channel-table slot instead. `broadcast_offer_channel` (the +advertised join token) is **always** inline regardless of path — it is the advertisement payload +and must carry the actual name/PSK. + +#### Configuring a multi-target broadcaster (two-step) + +Because a target's channel is a reference, configuring a multi-target broadcaster takes **two +admin writes**, in order: + +1. **Create/define each channel in the node's channel table** with the normal channel admin flow + (the same `set_channel` your app already uses for adding channels): + + ```text + AdminMessage.set_channel { index: 1, role: SECONDARY, + settings: { name: "NarrowSlow", psk: , channel_num: 0 } } + ``` + +2. **Write the beacon config**, pointing each target at the slot index from step 1: + + ```text + AdminMessage.set_module_config { mesh_beacon: { + flags = FLAG_BROADCAST_ENABLED + broadcast_targets = [ { preset: NARROW_SLOW, region: EU_N_868, channel_index: 1 } ] + } } + ``` + +Notes: + +- A target may **only** reference a channel that already exists locally — the node needs that + channel's key to encrypt the beacon. A `channel_index` that is out of range, or points at a + blank/unconfigured slot, is not an error: the beacon falls back to the node's **current/primary + channel** (its name, PSK, and slot) on the target preset/region. The channel name only defaults + to the preset's display name (e.g. `LongFast`) when the primary channel itself is unnamed — so + the fallback is "broadcast on my home channel," **not** a freshly-synthesised default-PSK channel + for that preset. +- `channel_index` must be `< MAX_NUM_CHANNELS` (8); the firmware clears it on write otherwise (see + §2.2 sanitise rules). This is the **only** check on write — the firmware does **not** verify that + the referenced slot is actually populated, because you may legitimately write the beacon config + before creating the channel. **Validating that a referenced channel exists is the client app's + responsibility.** A dangling reference doesn't error; it silently falls back to the preset's + default channel — so without a client-side check, the user can believe they're advertising + channel _X_ while the node is really transmitting on the preset default. Before writing, confirm + each `channel_index` maps to a configured `Channel`, and warn the user otherwise. +- **No automatic deduplication of channels.** Neither the beacon config nor the channel table + dedups by content: two `broadcast_targets` may carry the same `channel_index`, or different + indices whose slots hold identical settings, and `set_channel` will happily store two slots with + the same name/PSK. The broadcaster _does_ skip transmitting a target whose effective + preset/region/channel duplicates an earlier one in the same cycle (so a duplicated entry wastes + no airtime), but it does not rewrite or reject your config — keeping the target list free of + redundant entries is up to the client. +- The single-target path needs no separate `set_channel` step — its `broadcast_on_channel` is + written inline in the same beacon-config message. + +### 2.6 Quick reference + +| Concern | Value | +| ---------------------- | ---------------------------------------------------------------------------------------- | +| Port number | `MESH_BEACON_APP = 37` | +| Wire message | `meshtastic.MeshBeacon` | +| Config message | `ModuleConfig.MeshBeaconConfig` (variant tag 17) | +| On/off toggles | `flags` bitfield (`MeshBeaconConfig.Flags`) | +| Local config presence | `LocalModuleConfig.mesh_beacon` (tag 18) | +| Min broadcast interval | 3600 s (1 h) | +| Message max length | 100 bytes | +| Hop behaviour | Zero-hop (`hop_limit = 0`), never rebroadcast; `hop_start = 1` under `FLAG_LEGACY_SPLIT` | +| Auto-apply offers? | **Never** — client + user decide | +| Offer PSK | Public join token, not a secret | +| Disabled today | `broadcast_send_as_node` application | diff --git a/src/mesh/Channels.h b/src/mesh/Channels.h index 7392284f5..d1ae9b91b 100644 --- a/src/mesh/Channels.h +++ b/src/mesh/Channels.h @@ -105,6 +105,11 @@ class Channels bool setDefaultPresetCryptoForHash(ChannelHash channelHash); + /** + * Validate a channel, fixing any errors as needed + */ + meshtastic_Channel &fixupChannel(ChannelIndex chIndex); + int16_t getHash(ChannelIndex i) { return hashes[i]; } private: @@ -124,11 +129,6 @@ class Channels */ int16_t generateHash(ChannelIndex channelNum); - /** - * Validate a channel, fixing any errors as needed - */ - meshtastic_Channel &fixupChannel(ChannelIndex chIndex); - /** * Writes the default lora config */ diff --git a/src/mesh/Default.h b/src/mesh/Default.h index e802a0324..b38a68185 100644 --- a/src/mesh/Default.h +++ b/src/mesh/Default.h @@ -30,6 +30,7 @@ #define default_screen_on_secs IF_ROUTER(1, 60 * 10) #define default_node_info_broadcast_secs 3 * 60 * 60 #define default_neighbor_info_broadcast_secs 6 * 60 * 60 +#define default_mesh_beacon_min_broadcast_interval_secs 3600 #define min_node_info_broadcast_secs 60 * 60 // No regular broadcasts of more than once an hour #define min_neighbor_info_broadcast_secs 4 * 60 * 60 #define default_map_publish_interval_secs 60 * 60 diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index 05d36c8af..3a01aa357 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -1308,6 +1308,152 @@ void NodeDB::installDefaultModuleConfig() moduleConfig.ambient_lighting.green = (myNodeInfo.my_node_num & 0x00FF00) >> 8; moduleConfig.ambient_lighting.blue = myNodeInfo.my_node_num & 0x0000FF; +#if !MESHTASTIC_EXCLUDE_BEACON + moduleConfig.has_mesh_beacon = true; + // Default flags: listen on, broadcast off, legacy split on. + moduleConfig.mesh_beacon.flags = meshtastic_ModuleConfig_MeshBeaconConfig_Flags_FLAG_LISTEN_ENABLED | + meshtastic_ModuleConfig_MeshBeaconConfig_Flags_FLAG_LEGACY_SPLIT; +// Set or clear a single beacon flag bit from a USERPREFS boolean. +#define BEACON_APPLY_FLAG(enabled, flag) \ + do { \ + if (enabled) \ + moduleConfig.mesh_beacon.flags |= (uint32_t)(flag); \ + else \ + moduleConfig.mesh_beacon.flags &= ~(uint32_t)(flag); \ + } while (0) +#ifdef USERPREFS_MESH_BEACON_LISTEN_ENABLED + BEACON_APPLY_FLAG(USERPREFS_MESH_BEACON_LISTEN_ENABLED, meshtastic_ModuleConfig_MeshBeaconConfig_Flags_FLAG_LISTEN_ENABLED); +#endif +#ifdef USERPREFS_MESH_BEACON_BROADCAST_ENABLED + BEACON_APPLY_FLAG(USERPREFS_MESH_BEACON_BROADCAST_ENABLED, + meshtastic_ModuleConfig_MeshBeaconConfig_Flags_FLAG_BROADCAST_ENABLED); +#endif +#ifdef USERPREFS_MESH_BEACON_MESSAGE + strncpy(moduleConfig.mesh_beacon.broadcast_message, USERPREFS_MESH_BEACON_MESSAGE, + sizeof(moduleConfig.mesh_beacon.broadcast_message) - 1); + moduleConfig.mesh_beacon.broadcast_message[sizeof(moduleConfig.mesh_beacon.broadcast_message) - 1] = '\0'; +#endif +#ifdef USERPREFS_MESH_BEACON_INTERVAL_SECS + moduleConfig.mesh_beacon.broadcast_interval_secs = + (USERPREFS_MESH_BEACON_INTERVAL_SECS != 0 && + USERPREFS_MESH_BEACON_INTERVAL_SECS < default_mesh_beacon_min_broadcast_interval_secs) + ? default_mesh_beacon_min_broadcast_interval_secs + : USERPREFS_MESH_BEACON_INTERVAL_SECS; +#endif +#ifdef USERPREFS_MESH_BEACON_OFFER_PRESET + moduleConfig.mesh_beacon.has_broadcast_offer_preset = true; + moduleConfig.mesh_beacon.broadcast_offer_preset = USERPREFS_MESH_BEACON_OFFER_PRESET; +#endif +#ifdef USERPREFS_MESH_BEACON_OFFER_REGION + moduleConfig.mesh_beacon.broadcast_offer_region = USERPREFS_MESH_BEACON_OFFER_REGION; +#endif +#ifdef USERPREFS_MESH_BEACON_OFFER_CHANNEL_NAME + moduleConfig.mesh_beacon.has_broadcast_offer_channel = true; + strncpy(moduleConfig.mesh_beacon.broadcast_offer_channel.name, USERPREFS_MESH_BEACON_OFFER_CHANNEL_NAME, + sizeof(moduleConfig.mesh_beacon.broadcast_offer_channel.name) - 1); + moduleConfig.mesh_beacon.broadcast_offer_channel.name[sizeof(moduleConfig.mesh_beacon.broadcast_offer_channel.name) - 1] = + '\0'; +#endif +#ifdef USERPREFS_MESH_BEACON_OFFER_CHANNEL_PSK + moduleConfig.mesh_beacon.has_broadcast_offer_channel = true; + static const uint8_t beaconOfferPsk[] = USERPREFS_MESH_BEACON_OFFER_CHANNEL_PSK; + static_assert(sizeof(beaconOfferPsk) <= sizeof(moduleConfig.mesh_beacon.broadcast_offer_channel.psk.bytes), + "USERPREFS_MESH_BEACON_OFFER_CHANNEL_PSK exceeds the 32-byte channel PSK buffer"); + memcpy(moduleConfig.mesh_beacon.broadcast_offer_channel.psk.bytes, beaconOfferPsk, sizeof(beaconOfferPsk)); + moduleConfig.mesh_beacon.broadcast_offer_channel.psk.size = sizeof(beaconOfferPsk); +#endif +#ifdef USERPREFS_MESH_BEACON_ON_PRESET + moduleConfig.mesh_beacon.has_broadcast_on_preset = true; + moduleConfig.mesh_beacon.broadcast_on_preset = USERPREFS_MESH_BEACON_ON_PRESET; +#endif +#ifdef USERPREFS_MESH_BEACON_ON_REGION + moduleConfig.mesh_beacon.broadcast_on_region = USERPREFS_MESH_BEACON_ON_REGION; +#endif +#ifdef USERPREFS_MESH_BEACON_ON_CHANNEL_NAME + moduleConfig.mesh_beacon.has_broadcast_on_channel = true; + strncpy(moduleConfig.mesh_beacon.broadcast_on_channel.name, USERPREFS_MESH_BEACON_ON_CHANNEL_NAME, + sizeof(moduleConfig.mesh_beacon.broadcast_on_channel.name) - 1); + moduleConfig.mesh_beacon.broadcast_on_channel.name[sizeof(moduleConfig.mesh_beacon.broadcast_on_channel.name) - 1] = '\0'; +#endif +#ifdef USERPREFS_MESH_BEACON_ON_CHANNEL_PSK + moduleConfig.mesh_beacon.has_broadcast_on_channel = true; + static const uint8_t beaconOnPsk[] = USERPREFS_MESH_BEACON_ON_CHANNEL_PSK; + static_assert(sizeof(beaconOnPsk) <= sizeof(moduleConfig.mesh_beacon.broadcast_on_channel.psk.bytes), + "USERPREFS_MESH_BEACON_ON_CHANNEL_PSK exceeds the 32-byte channel PSK buffer"); + memcpy(moduleConfig.mesh_beacon.broadcast_on_channel.psk.bytes, beaconOnPsk, sizeof(beaconOnPsk)); + moduleConfig.mesh_beacon.broadcast_on_channel.psk.size = sizeof(beaconOnPsk); +#endif +#ifdef USERPREFS_MESH_BEACON_ON_CHANNEL_NUM + moduleConfig.mesh_beacon.has_broadcast_on_channel = true; + moduleConfig.mesh_beacon.broadcast_on_channel.channel_num = USERPREFS_MESH_BEACON_ON_CHANNEL_NUM; +#endif +#ifdef USERPREFS_MESH_BEACON_LEGACY_SPLIT + BEACON_APPLY_FLAG(USERPREFS_MESH_BEACON_LEGACY_SPLIT, meshtastic_ModuleConfig_MeshBeaconConfig_Flags_FLAG_LEGACY_SPLIT); +#endif +#undef BEACON_APPLY_FLAG +// Per-preset broadcast targets (up to 4). Each TARGET__* key bumps broadcast_targets_count as needed. +#define BEACON_TARGET_PRESET(N, VAL) \ + do { \ + if (moduleConfig.mesh_beacon.broadcast_targets_count < (N) + 1) \ + moduleConfig.mesh_beacon.broadcast_targets_count = (N) + 1; \ + moduleConfig.mesh_beacon.broadcast_targets[(N)].has_preset = true; \ + moduleConfig.mesh_beacon.broadcast_targets[(N)].preset = (VAL); \ + } while (0) +#define BEACON_TARGET_REGION(N, VAL) \ + do { \ + if (moduleConfig.mesh_beacon.broadcast_targets_count < (N) + 1) \ + moduleConfig.mesh_beacon.broadcast_targets_count = (N) + 1; \ + moduleConfig.mesh_beacon.broadcast_targets[(N)].region = (VAL); \ + } while (0) +// Target channel is referenced by index into the device's channel table (0..MAX_NUM_CHANNELS-1). +#define BEACON_TARGET_CH_INDEX(N, VAL) \ + do { \ + if (moduleConfig.mesh_beacon.broadcast_targets_count < (N) + 1) \ + moduleConfig.mesh_beacon.broadcast_targets_count = (N) + 1; \ + moduleConfig.mesh_beacon.broadcast_targets[(N)].has_channel_index = true; \ + moduleConfig.mesh_beacon.broadcast_targets[(N)].channel_index = (VAL); \ + } while (0) +#ifdef USERPREFS_MESH_BEACON_TARGET_0_PRESET + BEACON_TARGET_PRESET(0, USERPREFS_MESH_BEACON_TARGET_0_PRESET); +#endif +#ifdef USERPREFS_MESH_BEACON_TARGET_0_REGION + BEACON_TARGET_REGION(0, USERPREFS_MESH_BEACON_TARGET_0_REGION); +#endif +#ifdef USERPREFS_MESH_BEACON_TARGET_0_CHANNEL_INDEX + BEACON_TARGET_CH_INDEX(0, USERPREFS_MESH_BEACON_TARGET_0_CHANNEL_INDEX); +#endif +#ifdef USERPREFS_MESH_BEACON_TARGET_1_PRESET + BEACON_TARGET_PRESET(1, USERPREFS_MESH_BEACON_TARGET_1_PRESET); +#endif +#ifdef USERPREFS_MESH_BEACON_TARGET_1_REGION + BEACON_TARGET_REGION(1, USERPREFS_MESH_BEACON_TARGET_1_REGION); +#endif +#ifdef USERPREFS_MESH_BEACON_TARGET_1_CHANNEL_INDEX + BEACON_TARGET_CH_INDEX(1, USERPREFS_MESH_BEACON_TARGET_1_CHANNEL_INDEX); +#endif +#ifdef USERPREFS_MESH_BEACON_TARGET_2_PRESET + BEACON_TARGET_PRESET(2, USERPREFS_MESH_BEACON_TARGET_2_PRESET); +#endif +#ifdef USERPREFS_MESH_BEACON_TARGET_2_REGION + BEACON_TARGET_REGION(2, USERPREFS_MESH_BEACON_TARGET_2_REGION); +#endif +#ifdef USERPREFS_MESH_BEACON_TARGET_2_CHANNEL_INDEX + BEACON_TARGET_CH_INDEX(2, USERPREFS_MESH_BEACON_TARGET_2_CHANNEL_INDEX); +#endif +#ifdef USERPREFS_MESH_BEACON_TARGET_3_PRESET + BEACON_TARGET_PRESET(3, USERPREFS_MESH_BEACON_TARGET_3_PRESET); +#endif +#ifdef USERPREFS_MESH_BEACON_TARGET_3_REGION + BEACON_TARGET_REGION(3, USERPREFS_MESH_BEACON_TARGET_3_REGION); +#endif +#ifdef USERPREFS_MESH_BEACON_TARGET_3_CHANNEL_INDEX + BEACON_TARGET_CH_INDEX(3, USERPREFS_MESH_BEACON_TARGET_3_CHANNEL_INDEX); +#endif +#undef BEACON_TARGET_PRESET +#undef BEACON_TARGET_REGION +#undef BEACON_TARGET_CH_INDEX +#endif // !MESHTASTIC_EXCLUDE_BEACON + initModuleConfigIntervals(); } @@ -2758,6 +2904,9 @@ bool NodeDB::saveToDiskNoRetry(int saveWhat) moduleConfig.has_audio = true; moduleConfig.has_paxcounter = true; moduleConfig.has_statusmessage = true; +#if !MESHTASTIC_EXCLUDE_BEACON + moduleConfig.has_mesh_beacon = true; +#endif success &= saveProto(moduleConfigFileName, meshtastic_LocalModuleConfig_size, &meshtastic_LocalModuleConfig_msg, &moduleConfig); diff --git a/src/mesh/RadioLibInterface.cpp b/src/mesh/RadioLibInterface.cpp index daf98696e..3aff92831 100644 --- a/src/mesh/RadioLibInterface.cpp +++ b/src/mesh/RadioLibInterface.cpp @@ -8,6 +8,9 @@ #include "error.h" #include "main.h" #include "mesh-pb-constants.h" +#if !MESHTASTIC_EXCLUDE_BEACON +#include "modules/MeshBeaconModule.h" +#endif #include #include @@ -365,7 +368,15 @@ void RadioLibInterface::onNotify(uint32_t notification) switch (notification) { case ISR_TX: - handleTransmitInterrupt(); + handleTransmitInterrupt(); // completeSending() already restored the radio to the home config +#if !MESHTASTIC_EXCLUDE_BEACON + // Pre-switch the radio to the NEXT queued packet's beacon config (no-op for normal traffic). + // Not required for correctness — TRANSMIT_DELAY_COMPLETED would switch before CAD anyway — but + // doing it here lets the next beacon skip the switch-only delay cycle and, more importantly, + // keeps the post-TX listen window (and the CAD/LBT that follows) on the channel we're about to + // transmit on. Only engages when the next packet is itself a beacon — exactly when we want it. + MeshBeaconModule::reconfigureForBeaconTX(this, txQueue.getFront()); +#endif startReceive(); setTransmitDelay(); break; @@ -388,9 +399,27 @@ void RadioLibInterface::onNotify(uint32_t notification) if (delay_remaining > 0) { // There's still some delay pending on this packet, so resume waiting for it to elapse notifyLater(delay_remaining, TRANSMIT_DELAY_COMPLETED, false); +#if !MESHTASTIC_EXCLUDE_BEACON + } else if (MeshBeaconModule::beaconTxConfigInvalid(txp)) { + // The beacon's target radio config is invalid (bad preset/region, or an + // unlicensed node keying up on a ham-only region). Drop the packet — never + // transmit it on the current (home) config — and move on to the next queued packet. + LOG_DEBUG("Beacon: invalid TX radio config, dropping packet 0x%08x", txp->id); + meshtastic_MeshPacket *bad = txQueue.dequeue(); + MeshBeaconModule::clearTargetRadioSettings(bad); + packetPool.release(bad); + setTransmitDelay(); + } else if (MeshBeaconModule::reconfigureForBeaconTX(this, txp)) { + setTransmitDelay(); +#endif } else { if (isChannelActive()) { // check if there is currently a LoRa packet on the channel - startReceive(); // try receiving this packet, afterwards we'll be trying to transmit again +#if !MESHTASTIC_EXCLUDE_BEACON + if (!MeshBeaconModule::hasTargetRadioSettings(txp)) +#endif + { + startReceive(); // try receiving this packet, afterwards we'll be trying to transmit again + } setTransmitDelay(); } else { // Send any outgoing packets we have ready as fast as possible to keep the time between channel scan and @@ -522,6 +551,10 @@ void RadioLibInterface::completeSending() if (!isFromUs(p)) txRelay++; printPacket("Completed sending", p); +#if !MESHTASTIC_EXCLUDE_BEACON + MeshBeaconModule::clearTargetRadioSettings(p); + MeshBeaconModule::reconfigureForBeaconTX(this, nullptr); +#endif // We are done sending that packet, release it packetPool.release(p); @@ -682,6 +715,13 @@ bool RadioLibInterface::startSend(meshtastic_MeshPacket *txp) channel scan and actual transmit as low as possible to avoid collisions. */ if (disabled || !config.lora.tx_enabled) { LOG_WARN("Drop Tx packet because LoRa Tx disabled"); +#if !MESHTASTIC_EXCLUDE_BEACON + // This packet may have already triggered a beacon radio switch in TRANSMIT_DELAY_COMPLETED; + // since it never reaches completeSending() here, restore the radio so it isn't left on the + // beacon config (which would also break RX on the home channel). + MeshBeaconModule::clearTargetRadioSettings(txp); + MeshBeaconModule::reconfigureForBeaconTX(this, nullptr); +#endif packetPool.release(txp); return false; } else { diff --git a/src/mesh/Router.cpp b/src/mesh/Router.cpp index d52fa49f7..6f08e8cab 100644 --- a/src/mesh/Router.cpp +++ b/src/mesh/Router.cpp @@ -832,6 +832,19 @@ void Router::handleReceived(meshtastic_MeshPacket *p, RxSource src) skipHandle = true; } +#if !MESHTASTIC_EXCLUDE_BEACON + // Beacon listening is disabled: drop beacon packets so they are neither surfaced to the + // phone nor handled on-device (same pattern as the disabled neighbor-info case above). + if (p->which_payload_variant == meshtastic_MeshPacket_decoded_tag && + p->decoded.portnum == meshtastic_PortNum_MESH_BEACON_APP && + (!moduleConfig.has_mesh_beacon || + !(moduleConfig.mesh_beacon.flags & meshtastic_ModuleConfig_MeshBeaconConfig_Flags_FLAG_LISTEN_ENABLED))) { + LOG_DEBUG("Beacon listening is disabled, ignore beacon packet"); + cancelSending(p->from, p->id); + skipHandle = true; + } +#endif + bool shouldIgnoreNonstandardPorts = config.device.rebroadcast_mode == meshtastic_Config_DeviceConfig_RebroadcastMode_CORE_PORTNUMS_ONLY; #if USERPREFS_EVENT_MODE diff --git a/src/modules/AdminModule.cpp b/src/modules/AdminModule.cpp index 39ae6b788..e19ad3dbb 100644 --- a/src/modules/AdminModule.cpp +++ b/src/modules/AdminModule.cpp @@ -35,6 +35,9 @@ #ifdef MESHTASTIC_ENCRYPTED_STORAGE #include "security/EncryptedStorage.h" #endif +#if !MESHTASTIC_EXCLUDE_BEACON +#include "modules/MeshBeaconModule.h" +#endif #if !MESHTASTIC_EXCLUDE_MQTT #include "mqtt/MQTT.h" @@ -313,6 +316,17 @@ bool AdminModule::handleReceivedProtobuf(const meshtastic_MeshPacket &mp, meshta case meshtastic_AdminMessage_set_module_config_tag: LOG_DEBUG("Client set module config"); +#if !MESHTASTIC_EXCLUDE_BEACON + // broadcast_send_as_node: remote admins may only set this to their own node ID. + if (mp.from != 0 && r->set_module_config.which_payload_variant == meshtastic_ModuleConfig_mesh_beacon_tag) { + auto &b = r->set_module_config.payload_variant.mesh_beacon; + if (b.broadcast_send_as_node != 0 && b.broadcast_send_as_node != mp.from) { + LOG_WARN("Beacon: rejecting broadcast_send_as_node 0x%08x from node 0x%08x (must match sender)", + b.broadcast_send_as_node, mp.from); + b.broadcast_send_as_node = moduleConfig.mesh_beacon.broadcast_send_as_node; + } + } +#endif if (!handleSetModuleConfig(r->set_module_config)) { myReply = allocErrorResponse(meshtastic_Routing_Error_BAD_REQUEST, &mp); } @@ -1161,6 +1175,91 @@ bool AdminModule::handleSetModuleConfig(const meshtastic_ModuleConfig &c) moduleConfig.has_traffic_management = true; moduleConfig.traffic_management = c.payload_variant.traffic_management; break; +#if !MESHTASTIC_EXCLUDE_BEACON + case meshtastic_ModuleConfig_mesh_beacon_tag: { + LOG_INFO("Set module config: MeshBeacon"); + // Sanitize a local copy rather than const_cast-ing the const input (UB if a truly-const + // object is ever passed); the validated copy is assigned into moduleConfig below. + auto beaconCfg = c.payload_variant.mesh_beacon; + // Hard cap at 100 chars. + beaconCfg.broadcast_message[100] = '\0'; + // Enforce interval minimum (0 means unset/use default). + if (beaconCfg.broadcast_interval_secs != 0 && + beaconCfg.broadcast_interval_secs < default_mesh_beacon_min_broadcast_interval_secs) + beaconCfg.broadcast_interval_secs = default_mesh_beacon_min_broadcast_interval_secs; + // Validate broadcast_on_preset against broadcast_on_region (or current region if unset). + if (beaconCfg.has_broadcast_on_preset) { + meshtastic_Config_LoRaConfig probe = config.lora; + probe.use_preset = true; + probe.modem_preset = beaconCfg.broadcast_on_preset; + if (beaconCfg.broadcast_on_region != meshtastic_Config_LoRaConfig_RegionCode_UNSET) + probe.region = beaconCfg.broadcast_on_region; + if (!RadioInterface::validateConfigLora(probe)) { + LOG_WARN("Beacon: broadcast_on_preset %d invalid for region, clearing", beaconCfg.broadcast_on_preset); + beaconCfg.has_broadcast_on_preset = false; + beaconCfg.has_broadcast_on_channel = false; + } + } + // Validate broadcast_offer_preset against broadcast_offer_region (or current region if unset). + if (beaconCfg.has_broadcast_offer_preset) { + meshtastic_Config_LoRaConfig probe = config.lora; + probe.use_preset = true; + probe.modem_preset = beaconCfg.broadcast_offer_preset; + if (beaconCfg.broadcast_offer_region != meshtastic_Config_LoRaConfig_RegionCode_UNSET) + probe.region = beaconCfg.broadcast_offer_region; + if (!RadioInterface::validateConfigLora(probe)) { + LOG_WARN("Beacon: broadcast_offer_preset %d invalid for region, clearing", beaconCfg.broadcast_offer_preset); + beaconCfg.has_broadcast_offer_preset = false; + } + } + // Validate broadcast_offer_region is a known region code. + if (beaconCfg.broadcast_offer_region != meshtastic_Config_LoRaConfig_RegionCode_UNSET) { + const RegionInfo *r = getRegion(beaconCfg.broadcast_offer_region); + if (r->code != beaconCfg.broadcast_offer_region) { + LOG_WARN("Beacon: broadcast_offer_region %d invalid, clearing", beaconCfg.broadcast_offer_region); + beaconCfg.broadcast_offer_region = meshtastic_Config_LoRaConfig_RegionCode_UNSET; + } + } + // Validate each multi-target entry the same way as the single-target broadcast_on_* fields, + // so a bad preset/region is cleared on write rather than relying on the runtime TX drop. + for (pb_size_t i = 0; i < beaconCfg.broadcast_targets_count; i++) { + auto &t = beaconCfg.broadcast_targets[i]; + // Region must be a known region code (UNSET = use running config at TX time). + if (t.region != meshtastic_Config_LoRaConfig_RegionCode_UNSET) { + const RegionInfo *r = getRegion(t.region); + if (r->code != t.region) { + LOG_WARN("Beacon: broadcast_targets[%u] region %d invalid, clearing", i, t.region); + t.region = meshtastic_Config_LoRaConfig_RegionCode_UNSET; + } + } + // Preset must be valid for the target region (or current region if unset). + if (t.has_preset) { + meshtastic_Config_LoRaConfig probe = config.lora; + probe.use_preset = true; + probe.modem_preset = t.preset; + if (t.region != meshtastic_Config_LoRaConfig_RegionCode_UNSET) + probe.region = t.region; + if (!RadioInterface::validateConfigLora(probe)) { + LOG_WARN("Beacon: broadcast_targets[%u] preset %d invalid for region, clearing", i, t.preset); + t.has_preset = false; + t.has_channel_index = false; + } + } + // channel_index must reference a real channel-table slot. + if (t.has_channel_index && t.channel_index >= MAX_NUM_CHANNELS) { + LOG_WARN("Beacon: broadcast_targets[%u] channel_index %u out of range, clearing", i, t.channel_index); + t.has_channel_index = false; + } + } + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon = beaconCfg; + shouldReboot = false; + // Payload content changed — invalidate the broadcaster's cache. + if (meshBeaconBroadcastModule) + meshBeaconBroadcastModule->invalidateCache(); + break; + } +#endif } saveChanges(SEGMENT_MODULECONFIG, shouldReboot); return true; diff --git a/src/modules/AdminModule.h b/src/modules/AdminModule.h index 1d5c64423..d072ab2fe 100644 --- a/src/modules/AdminModule.h +++ b/src/modules/AdminModule.h @@ -64,7 +64,11 @@ class AdminModule : public ProtobufModule, public Obser protected: void handleSetConfig(const meshtastic_Config &c, bool fromOthers); +#ifdef PIO_UNIT_TESTING + protected: +#else private: +#endif bool handleSetModuleConfig(const meshtastic_ModuleConfig &c); void handleSetChannel(); diff --git a/src/modules/MeshBeaconModule.cpp b/src/modules/MeshBeaconModule.cpp new file mode 100644 index 000000000..8a86aebaf --- /dev/null +++ b/src/modules/MeshBeaconModule.cpp @@ -0,0 +1,611 @@ +#include "MeshBeaconModule.h" +#include "Default.h" +#include "DisplayFormatters.h" +#include "NodeDB.h" +#include "RTC.h" +#include "RadioInterface.h" +#include "Router.h" +#include "TransmitHistory.h" +#include "configuration.h" +#include "main.h" +#include +#include + +// Static members +meshtastic_Config_LoRaConfig_ModemPreset MeshBeaconModule::originalModemPreset; +uint16_t MeshBeaconModule::originalLoraChannel; +meshtastic_Config_LoRaConfig_RegionCode MeshBeaconModule::originalRegion; +meshtastic_ChannelSettings MeshBeaconModule::originalPrimaryChannel; + +static MeshBeaconModule_TargetRadioSettings targetRadioSettings[8]; + +static bool getTargetRadioSettings(const meshtastic_MeshPacket *p, meshtastic_Config_LoRaConfig_ModemPreset *preset, + uint16_t *slot, bool *legacyHopOverride = nullptr, + meshtastic_Config_LoRaConfig_RegionCode *region = nullptr, bool *has_channel = nullptr, + meshtastic_ChannelSettings *channel = nullptr) +{ + if (!p) + return false; + for (const auto &entry : targetRadioSettings) { + if (entry.inUse && entry.id == p->id) { + if (preset) + *preset = entry.preset; + if (slot) + *slot = entry.slot; + if (legacyHopOverride) + *legacyHopOverride = entry.legacyHopOverride; + if (region) + *region = entry.region; + if (has_channel) + *has_channel = entry.has_channel; + if (channel && entry.has_channel) + *channel = entry.channel; + return true; + } + } + return false; +} + +// --------------------------------------------------------------------------- +// MeshBeaconModule base +// --------------------------------------------------------------------------- + +MeshBeaconModule::MeshBeaconModule() +{ + originalModemPreset = config.lora.modem_preset; + originalLoraChannel = config.lora.channel_num; + originalRegion = config.lora.region; + originalPrimaryChannel = channels.getPrimary(); +} + +void MeshBeaconModule::setTargetRadioSettings(const meshtastic_MeshPacket *p, meshtastic_Config_LoRaConfig_ModemPreset preset, + uint16_t slot, bool legacyHopOverride, + meshtastic_Config_LoRaConfig_RegionCode region, bool has_channel, + const meshtastic_ChannelSettings *channel) +{ + if (!p) + return; + MeshBeaconModule_TargetRadioSettings *target = nullptr; + for (auto &entry : targetRadioSettings) { + if (entry.inUse && entry.id == p->id) { + target = &entry; + break; + } + if (!target && !entry.inUse) + target = &entry; + } + if (!target) + target = &targetRadioSettings[0]; + target->inUse = true; + target->id = p->id; + target->preset = preset; + target->slot = slot; + target->legacyHopOverride = legacyHopOverride; + target->region = region; + target->has_channel = has_channel; + if (has_channel && channel) + target->channel = *channel; +} + +bool MeshBeaconModule::hasTargetRadioSettings(const meshtastic_MeshPacket *p) +{ + return getTargetRadioSettings(p, nullptr, nullptr); +} + +void MeshBeaconModule::clearTargetRadioSettings(const meshtastic_MeshPacket *p) +{ + if (!p) + return; + for (auto &entry : targetRadioSettings) { + if (entry.inUse && entry.id == p->id) { + entry.inUse = false; + return; + } + } +} + +bool MeshBeaconModule::beaconTxConfigInvalid(const meshtastic_MeshPacket *p) +{ + meshtastic_Config_LoRaConfig_ModemPreset preset; + meshtastic_Config_LoRaConfig_RegionCode sidecarRegion = meshtastic_Config_LoRaConfig_RegionCode_UNSET; + if (!getTargetRadioSettings(p, &preset, nullptr, nullptr, &sidecarRegion)) + return false; // not a beacon-switch packet — nothing to validate, normal traffic unaffected + + const meshtastic_Config_LoRaConfig_RegionCode region = + (sidecarRegion != meshtastic_Config_LoRaConfig_RegionCode_UNSET) ? sidecarRegion : config.lora.region; + + // An unlicensed node must never key up on a ham-only (licensed-only) region. The reverse is + // allowed: a licensed (ham) node may operate in a non-ham region — and the switch only touches + // preset/region/channel, never owner.is_licensed, so it cannot deactivate licensed mode. + const RegionInfo *r = getRegion(region); + if (r && r->profile->licensedOnly && !owner.is_licensed) + return true; + + // Preset must be valid for the target region. + meshtastic_Config_LoRaConfig probe = config.lora; + probe.use_preset = true; + probe.modem_preset = preset; + probe.region = region; + return !RadioInterface::validateConfigLora(probe); +} + +meshtastic_ChannelSettings MeshBeaconModule::beaconChannelSettings(const meshtastic_ChannelSettings &base, + meshtastic_Config_LoRaConfig_ModemPreset preset, + const meshtastic_ChannelSettings *overrideChannel) +{ + meshtastic_ChannelSettings ch = base; + if (overrideChannel) { + ch.channel_num = overrideChannel->channel_num; + if (overrideChannel->name[0] != '\0') + strncpy(ch.name, overrideChannel->name, sizeof(ch.name) - 1); + if (overrideChannel->psk.size > 0) + ch.psk = overrideChannel->psk; + } + // If no usable name survived (no override, or a blank-named one), default to the preset's + // display name so the beacon channel is identifiable rather than borrowing the primary's name. + if (ch.name[0] == '\0') + strncpy(ch.name, DisplayFormatters::getModemPresetDisplayName(preset, false, true), sizeof(ch.name) - 1); + ch.name[sizeof(ch.name) - 1] = '\0'; + return ch; +} + +bool MeshBeaconModule::reconfigureForBeaconTX(RadioInterface *iface, meshtastic_MeshPacket *p) +{ + // True while a beacon radio switch is in effect and still needs undoing. We track the switch + // explicitly rather than inferring it from "live config differs from the snapshot", because that + // heuristic both missed cases (a channel name/PSK swap that left preset/slot/region unchanged would + // never be restored) and fired falsely (a legitimate non-beacon channel edit would be reverted on + // the next TX). With the flag the restore fires for ANY field we changed and only when we changed + // it — including on TX-failure paths, which route through this same restore call. + static bool radioSwitched = false; + + meshtastic_ChannelSettings *primaryCh = &channels.getByIndex(channels.getPrimaryIndex()).settings; + meshtastic_Config_LoRaConfig_ModemPreset targetPreset; + uint16_t targetSlot; + + const auto channelDiffers = [&](const meshtastic_ChannelSettings &target) { + return strncmp(primaryCh->name, target.name, sizeof(primaryCh->name)) != 0 || primaryCh->psk.size != target.psk.size || + memcmp(primaryCh->psk.bytes, target.psk.bytes, primaryCh->psk.size) != 0 || + primaryCh->channel_num != target.channel_num; + }; + + bool legacyHopOverride = false; + meshtastic_Config_LoRaConfig_RegionCode sidecarRegion = meshtastic_Config_LoRaConfig_RegionCode_UNSET; + bool sidecarHasChannel = false; + meshtastic_ChannelSettings sidecarChannel = {}; + if (p && getTargetRadioSettings(p, &targetPreset, &targetSlot, &legacyHopOverride, &sidecarRegion, &sidecarHasChannel, + &sidecarChannel)) { + + // Legacy compatibility: older firmware (pre-v2.7.20) drops hop_start==0 packets via the + // pre-hop check before decryption, so they can't see has_bitfield to validate them. + // Setting hop_start=1 (with hop_limit remaining 0) makes the packet pass the old check + // while still being zero-hop (hop_limit=0 prevents any rebroadcast). + if (legacyHopOverride) + p->hop_start = 1; + + const meshtastic_Config_LoRaConfig_RegionCode targetRegion = + (sidecarRegion != meshtastic_Config_LoRaConfig_RegionCode_UNSET) ? sidecarRegion : config.lora.region; + const meshtastic_ChannelSettings *overrideCh = sidecarHasChannel ? &sidecarChannel : nullptr; + + meshtastic_ChannelSettings targetChannel = beaconChannelSettings(*primaryCh, targetPreset, overrideCh); + + if (targetPreset == config.lora.modem_preset && targetSlot == config.lora.channel_num && + targetRegion == config.lora.region && !channelDiffers(targetChannel)) + return false; + + // Guard: never key up on an invalid target config — bad preset for the region, or an + // unlicensed node keying up on a ham-only region. Refuse the switch here so we never + // transmit on it; the radio driver drops the packet outright (see RadioLibInterface, + // beaconTxConfigInvalid) rather than letting it fall through onto the current config. + if (beaconTxConfigInvalid(p)) { + LOG_DEBUG("Beacon: target preset %d/region %d invalid (or ham mismatch), not switching", targetPreset, targetRegion); + return false; + } + + // Snapshot current (non-beacon) settings so we restore to the latest config. Skip while a + // switch is already active, so a second switch before the restore can't capture the beacon + // config as the "home" we later restore to. + if (!radioSwitched) { + originalModemPreset = config.lora.modem_preset; + originalLoraChannel = config.lora.channel_num; + originalRegion = config.lora.region; + originalPrimaryChannel = *primaryCh; + } + + LOG_INFO("Beacon: switch radio for packet 0x%08x to preset=%d slot=%u region=%d", p->id, targetPreset, targetSlot, + targetRegion); + config.lora.modem_preset = targetPreset; + config.lora.channel_num = targetSlot; + if (targetRegion != config.lora.region) + config.lora.region = targetRegion; + *primaryCh = targetChannel; + + channels.fixupChannel(channels.getPrimaryIndex()); + p->channel = channels.getHash(channels.getPrimaryIndex()); + iface->reconfigure(); + radioSwitched = true; + return true; + + } else if ((!p || !getTargetRadioSettings(p, nullptr, nullptr)) && radioSwitched) { + + LOG_INFO("Beacon: restoring radio config after beacon TX"); + config.lora.modem_preset = originalModemPreset; + config.lora.channel_num = originalLoraChannel; + config.lora.region = originalRegion; + *primaryCh = originalPrimaryChannel; + primaryCh->name[sizeof(primaryCh->name) - 1] = '\0'; + + channels.fixupChannel(channels.getPrimaryIndex()); + iface->reconfigure(); + radioSwitched = false; + return true; + } + return false; +} + +// --------------------------------------------------------------------------- +// MeshBeaconBroadcastModule +// --------------------------------------------------------------------------- + +MeshBeaconBroadcastModule *meshBeaconBroadcastModule; + +MeshBeaconBroadcastModule::MeshBeaconBroadcastModule() + : MeshBeaconModule(), ProtobufModule("beacon_tx", meshtastic_PortNum_MESH_BEACON_APP, &meshtastic_MeshBeacon_msg), + concurrency::OSThread("MeshBeaconBroadcast") +{ + setIntervalFromNow(setStartDelay()); +} + +void MeshBeaconBroadcastModule::rebuildCache() +{ + const auto &bcfg = moduleConfig.mesh_beacon; + meshtastic_MeshBeacon beacon = meshtastic_MeshBeacon_init_zero; + strncpy(beacon.message, bcfg.broadcast_message, sizeof(beacon.message) - 1); + if (bcfg.has_broadcast_offer_channel) { + beacon.has_offer_channel = true; + beacon.offer_channel = bcfg.broadcast_offer_channel; + // PSK is included intentionally: this beacon is a public join-invitation. + // The offered channel is not secret — the PSK here is a convenience token, + // not a security boundary. Operators who want a private channel must + // distribute the PSK out-of-band and leave offer_channel unset. + } + beacon.has_offer_preset = bcfg.has_broadcast_offer_preset; + beacon.offer_preset = bcfg.broadcast_offer_preset; + beacon.offer_region = bcfg.broadcast_offer_region; + // Note: an empty config legitimately encodes to 0 bytes, and pb_encode_to_bytes can't distinguish + // that from a (here effectively impossible — buffer is max-sized) failure, so we always clear the + // dirty flag. The combined send is gated on payloadCacheSize > 0, so an empty payload is never TX'd. + payloadCacheSize = (pb_size_t)pb_encode_to_bytes(payloadCache, sizeof(payloadCache), &meshtastic_MeshBeacon_msg, &beacon); + payloadCacheDirty = false; + LOG_DEBUG("Beacon: payload cache rebuilt (%u bytes)", payloadCacheSize); +} + +void MeshBeaconBroadcastModule::sendBeaconPacket(meshtastic_MeshPacket *p, meshtastic_Config_LoRaConfig_ModemPreset targetPreset, + bool has_channel, const meshtastic_ChannelSettings *overrideChannel) +{ + const bool cryptoOverride = + has_channel && overrideChannel && (overrideChannel->name[0] != '\0' || overrideChannel->psk.size > 0); + if (!cryptoOverride) { + router->send(p); + return; + } + + // perhapsEncode() keys encryption (and the channel-hash hint) off the PRIMARY channel slot, and + // the radio-thread channel switch only happens AFTER encryption — so a beacon on an override + // channel would otherwise be encrypted with the PRIMARY PSK, not the beacon channel's. Install the + // beacon channel into the primary slot for the synchronous duration of send(), then restore. + // Meshtastic threading is cooperative (no preemption between the swap and restore). + meshtastic_Channel &primary = channels.getByIndex(channels.getPrimaryIndex()); + const meshtastic_ChannelSettings saved = primary.settings; + primary.settings = beaconChannelSettings(saved, targetPreset, overrideChannel); + channels.fixupChannel(channels.getPrimaryIndex()); + + router->send(p); // encrypts with the beacon channel's key and stamps its hash + + primary.settings = saved; + channels.fixupChannel(channels.getPrimaryIndex()); +} + +void MeshBeaconBroadcastModule::sendBeacon() +{ + const auto &bcfg = moduleConfig.mesh_beacon; + + const bool hasText = bcfg.broadcast_message[0] != '\0'; + const bool hasRadioContent = bcfg.has_broadcast_offer_preset || bcfg.has_broadcast_offer_channel || + (bcfg.broadcast_offer_region != meshtastic_Config_LoRaConfig_RegionCode_UNSET); + + if (!hasText && !hasRadioContent) { + LOG_DEBUG("Beacon: nothing to send (empty message, no offer), skipping"); + return; + } + + // Stamp common fields shared by every outgoing beacon packet. + const auto stampPacket = [&](meshtastic_MeshPacket *p) { + p->to = NODENUM_BROADCAST; + p->from = nodeDB->getNodeNum(); + // broadcast_send_as_node: commented out pending further review. + // Spoof notes preserved for when this is re-enabled: + // broadcast_send_as_node overrides the source NodeNum. NOTE: this is a *node-ID* spoof + // only — it rewrites the 'from' field but does NOT forge any signature. Once 'from' is + // not us, the packet is no longer isFromUs(), so Router::perhapsEncode() skips XEdDSA + // signing and receivers get an unsigned packet attributed to another node. + // When broadcast_send_as_node == 0 the beacon is genuinely from us and Router::perhapsEncode() + // signs it under the same XEdDSA broadcast policy as normal channel messages. + // When broadcast_send_as_node rewrites p->from, perhapsEncode() sees isFromUs()=false and + // skips setting has_bitfield — must be set explicitly so receivers can classify hop_start + // correctly and so ok_to_mqtt is honoured on the spoofed packet. + // if (bcfg.broadcast_send_as_node != 0) { + // p->from = bcfg.broadcast_send_as_node; + // p->decoded.has_bitfield = true; + // p->decoded.bitfield |= (config.lora.config_ok_to_mqtt << BITFIELD_OK_TO_MQTT_SHIFT); + // } + p->hop_limit = 0; // all beacon packets are zero hopped to limit spamming. + p->priority = meshtastic_MeshPacket_Priority_BACKGROUND; + p->want_ack = false; + p->rx_time = getValidTime(RTCQualityFromNet); + }; + + // ── Packet type decisions ──────────────────────────────────────────────── + // + // FLAG_LEGACY_SPLIT: when both text and offer are present, send TWO packets — A) + // MESH_BEACON_APP (offer only) and B) TEXT_MESSAGE_APP (text only) — both on the SAME beacon + // radio settings, so nodes that only decode TEXT_MESSAGE_APP still receive the text. Otherwise a + // single packet is sent (offer-only, text-only, or the combined offer+text path). + // + // These are independent decisions, NOT a mutually-exclusive if/else chain: the split + // case must emit both A and B. Conditions are spelled out as named booleans to avoid + // the && / || precedence trap (and a prior bug where the split case dropped the text). + const bool legacySplit = bcfg.flags & MESH_BEACON_FLAG_LEGACY_SPLIT; + const bool splitBoth = legacySplit && hasRadioContent && hasText; + const bool sendOfferOnly = splitBoth || (hasRadioContent && !hasText); + const bool sendTextOnly = splitBoth || (!hasRadioContent && hasText); + const bool sendCombined = !legacySplit && hasRadioContent && hasText; + + // Build offer payload once — shared across all targets. + uint8_t offerBuf[meshtastic_MeshBeacon_size] = {}; + pb_size_t offerSize = 0; + if (sendOfferOnly) { + meshtastic_MeshBeacon offerOnly = meshtastic_MeshBeacon_init_zero; + if (bcfg.has_broadcast_offer_channel) { + offerOnly.has_offer_channel = true; + offerOnly.offer_channel = bcfg.broadcast_offer_channel; + } + offerOnly.has_offer_preset = bcfg.has_broadcast_offer_preset; + offerOnly.offer_preset = bcfg.broadcast_offer_preset; + offerOnly.offer_region = bcfg.broadcast_offer_region; + offerSize = (pb_size_t)pb_encode_to_bytes(offerBuf, sizeof(offerBuf), &meshtastic_MeshBeacon_msg, &offerOnly); + if (offerSize == 0) + LOG_WARN("Beacon: offer encode failed, skipping offer packet(s)"); + } + if (sendCombined && payloadCacheDirty) + rebuildCache(); + + // ── Per-target loop ────────────────────────────────────────────────────── + // + // If broadcast_targets is populated, iterate over those. Otherwise use the single-target + // broadcast_on_preset / broadcast_on_region / broadcast_on_channel fields. The two paths are + // equal options; they differ only in how the TX channel is named (single-target embeds a + // ChannelSettings inline; a target references a channel-table slot by channel_index). + struct EffTarget { + meshtastic_Config_LoRaConfig_ModemPreset preset; + uint16_t slot; + meshtastic_Config_LoRaConfig_RegionCode region; + bool has_channel; + meshtastic_ChannelSettings channel; + }; + + const bool useTargetList = bcfg.broadcast_targets_count > 0; + const int targetCount = useTargetList ? (int)bcfg.broadcast_targets_count : 1; + + // Dedup state: the beacon payload is identical across targets, so two targets that resolve to + // the same effective radio config (preset + resolved region + channel) would just re-broadcast + // the same packet — wasted airtime and a redundant radio switch each. We skip the later one. + // Keyed on the *resolved* values so an explicit "current region" dedups against an UNSET one. + EffTarget sent[4]; + meshtastic_Config_LoRaConfig_RegionCode sentRegion[4]; + int sentCount = 0; + const auto sameEffectiveTarget = [](const EffTarget &a, meshtastic_Config_LoRaConfig_RegionCode ar, const EffTarget &b, + meshtastic_Config_LoRaConfig_RegionCode br) { + if (a.preset != b.preset || ar != br || a.has_channel != b.has_channel) + return false; + if (!a.has_channel) + return true; // both fall back to the default channel for the (same) preset + return a.slot == b.slot && strncmp(a.channel.name, b.channel.name, sizeof(a.channel.name)) == 0 && + a.channel.psk.size == b.channel.psk.size && + memcmp(a.channel.psk.bytes, b.channel.psk.bytes, a.channel.psk.size) == 0; + }; + + for (int ti = 0; ti < targetCount; ti++) { + EffTarget tgt = {}; + if (useTargetList) { + const auto &bt = bcfg.broadcast_targets[ti]; + tgt.preset = bt.has_preset ? bt.preset : config.lora.modem_preset; + tgt.region = bt.region; + // Resolve the channel from the device's channel table by index. A slot is only usable + // if it is actually configured (has a name or PSK — its key is needed to encrypt). An + // out-of-range index, or a blank slot, falls back to the default channel for the target + // preset (see beaconChannelSettings), exactly as an unset channel_index would. + tgt.has_channel = false; + tgt.slot = config.lora.channel_num; + if (bt.has_channel_index) { + if (bt.channel_index >= (uint32_t)channels.getNumChannels()) { + LOG_WARN("Beacon: target %d channel_index %u out of range, using default channel for preset", ti, + bt.channel_index); + } else { + const meshtastic_ChannelSettings &cs = channels.getByIndex(bt.channel_index).settings; + if (cs.name[0] != '\0' || cs.psk.size > 0) { + tgt.has_channel = true; + tgt.channel = cs; + tgt.slot = cs.channel_num; + } else { + LOG_DEBUG("Beacon: target %d channel_index %u is a blank slot, using default channel for preset", ti, + bt.channel_index); + } + } + } + } else { + tgt.preset = bcfg.has_broadcast_on_preset ? bcfg.broadcast_on_preset : config.lora.modem_preset; + tgt.region = bcfg.broadcast_on_region; + tgt.has_channel = bcfg.has_broadcast_on_channel; + if (tgt.has_channel) + tgt.channel = bcfg.broadcast_on_channel; + tgt.slot = tgt.has_channel ? bcfg.broadcast_on_channel.channel_num : config.lora.channel_num; + } + + // Skip a target whose effective radio config duplicates one already sent this cycle. + const meshtastic_Config_LoRaConfig_RegionCode resolvedRegion = + (tgt.region != meshtastic_Config_LoRaConfig_RegionCode_UNSET) ? tgt.region : config.lora.region; + bool duplicate = false; + for (int si = 0; si < sentCount; si++) { + if (sameEffectiveTarget(tgt, resolvedRegion, sent[si], sentRegion[si])) { + duplicate = true; + break; + } + } + if (duplicate) { + LOG_DEBUG("Beacon: target %d duplicates an earlier target's radio config, skipping", ti); + continue; + } + sent[sentCount] = tgt; + sentRegion[sentCount] = resolvedRegion; + sentCount++; + + const bool channelOverrideConfigured = tgt.has_channel && (tgt.channel.name[0] != '\0' || tgt.channel.psk.size > 0 || + tgt.channel.channel_num != config.lora.channel_num); + const bool presetDiffers = + (tgt.preset != config.lora.modem_preset) || + (tgt.region != meshtastic_Config_LoRaConfig_RegionCode_UNSET && tgt.region != config.lora.region) || + channelOverrideConfigured; + const meshtastic_ChannelSettings *chPtr = tgt.has_channel ? &tgt.channel : nullptr; + + const auto applyTarget = [&](meshtastic_MeshPacket *p) { + if (presetDiffers || legacySplit) + setTargetRadioSettings(p, tgt.preset, tgt.slot, legacySplit, tgt.region, tgt.has_channel, chPtr); + sendBeaconPacket(p, tgt.preset, tgt.has_channel, chPtr); + }; + + if (sendOfferOnly && offerSize > 0) { + meshtastic_MeshPacket *pA = allocDataPacket(); + if (!pA) { + LOG_WARN("Beacon: failed to allocate split-A packet (target %d)", ti); + return; + } + memcpy(pA->decoded.payload.bytes, offerBuf, offerSize); + pA->decoded.payload.size = offerSize; + pA->decoded.portnum = meshtastic_PortNum_MESH_BEACON_APP; + stampPacket(pA); + LOG_INFO("Beacon: split-A MESH_BEACON_APP (offer only) from=0x%08x target=%d", pA->from, ti); + applyTarget(pA); + } + + if (sendTextOnly) { + meshtastic_MeshPacket *pB = allocDataPacket(); + if (!pB) { + LOG_WARN("Beacon: failed to allocate split-B packet (target %d)", ti); + return; + } + pb_size_t msgLen = (pb_size_t)strnlen(bcfg.broadcast_message, sizeof(bcfg.broadcast_message) - 1); + memcpy(pB->decoded.payload.bytes, bcfg.broadcast_message, msgLen); + pB->decoded.payload.size = msgLen; + pB->decoded.portnum = meshtastic_PortNum_TEXT_MESSAGE_APP; + stampPacket(pB); + LOG_INFO("Beacon: split-B TEXT_MESSAGE_APP msg='%.40s' from=0x%08x target=%d", bcfg.broadcast_message, pB->from, ti); + applyTarget(pB); + } + + if (sendCombined && payloadCacheSize > 0) { + meshtastic_MeshPacket *p = allocDataPacket(); + if (!p) { + LOG_WARN("Beacon: failed to allocate beacon packet (target %d)", ti); + return; + } + memcpy(p->decoded.payload.bytes, payloadCache, payloadCacheSize); + p->decoded.payload.size = payloadCacheSize; + p->decoded.portnum = meshtastic_PortNum_MESH_BEACON_APP; + stampPacket(p); + LOG_INFO("Beacon: MESH_BEACON_APP offer+msg from=0x%08x msg='%.40s' target=%d", p->from, bcfg.broadcast_message, ti); + applyTarget(p); + } + } +} + +int32_t MeshBeaconBroadcastModule::runOnce() +{ + const auto &bcfg = moduleConfig.mesh_beacon; + const uint32_t intervalSecs = + Default::getConfiguredOrDefault(bcfg.broadcast_interval_secs, default_mesh_beacon_min_broadcast_interval_secs); + const uint32_t intervalMs = + Default::getConfiguredOrMinimumValue(intervalSecs, default_mesh_beacon_min_broadcast_interval_secs) * 1000; + + if ((bcfg.flags & MESH_BEACON_FLAG_BROADCAST_ENABLED) && airTime->isTxAllowedAirUtil() && + config.device.role != meshtastic_Config_DeviceConfig_Role_CLIENT_HIDDEN) { + // Throttle against the reboot-safe transmit history (mirrors NodeInfoModule): skip if we + // broadcast within the interval, even across a reboot. 0 = never sent → send now. + uint32_t lastSent = transmitHistory ? transmitHistory->getLastSentToMeshMillis(meshtastic_PortNum_MESH_BEACON_APP) : 0; + if (lastSent == 0 || !Throttle::isWithinTimespanMs(lastSent, intervalMs)) { + // Record the send BEFORE transmitting: the LoRa TX is a high-current event that can + // brown out a marginal supply, and if that reboots us mid-transmit we still want the + // "sent" marker persisted so we don't re-broadcast immediately on every boot. + if (transmitHistory) + transmitHistory->setLastSentToMesh(meshtastic_PortNum_MESH_BEACON_APP); + sendBeacon(); + } + } + + return static_cast(intervalMs); +} + +// --------------------------------------------------------------------------- +// MeshBeaconListenerModule +// --------------------------------------------------------------------------- + +MeshBeaconListenerModule *meshBeaconListenerModule; +MeshBeaconListenerModule::BeaconOffer MeshBeaconListenerModule::lastReceivedOffer; + +MeshBeaconListenerModule::MeshBeaconListenerModule() + : ProtobufModule("beacon_listen", meshtastic_PortNum_MESH_BEACON_APP, &meshtastic_MeshBeacon_msg) +{ + lastReceivedOffer = {}; +} + +bool MeshBeaconListenerModule::wantPacket(const meshtastic_MeshPacket *p) +{ + return moduleConfig.has_mesh_beacon && (moduleConfig.mesh_beacon.flags & MESH_BEACON_FLAG_LISTEN_ENABLED) && + p->decoded.portnum == meshtastic_PortNum_MESH_BEACON_APP; +} + +bool MeshBeaconListenerModule::handleReceivedProtobuf(const meshtastic_MeshPacket &mp, meshtastic_MeshBeacon *b) +{ + const bool hasOfferContent = + b && (b->has_offer_channel || b->offer_region != meshtastic_Config_LoRaConfig_RegionCode_UNSET || b->has_offer_preset); + const pb_size_t msgLen = b ? (pb_size_t)strnlen(b->message, sizeof(b->message) - 1) : 0; + const bool hasText = msgLen > 0; + if (!b || (!hasText && !hasOfferContent)) + return false; + + // NOTE: we deliberately do NOT unwrap the text into a synthesized TEXT_MESSAGE_APP for the + // phone. The original MESH_BEACON_APP packet already flows to the client (we return CONTINUE), + // so a beacon-aware client renders `message` directly — injecting a copy would only duplicate + // it. Broadcasters that need non-beacon-aware clients to see the text use FLAG_LEGACY_SPLIT, + // which sends a real TEXT_MESSAGE_APP over RF. We also do not fire EVENT_RECEIVED_MSG: a beacon + // is an advisory broadcast, not a personal message, and must not wake the device from sleep. + if (hasText) + LOG_INFO("Beacon: received from 0x%08x: '%.40s'", mp.from, b->message); + + // Cache any offer for the client app — never auto-applied. + if (hasOfferContent) { + lastReceivedOffer.valid = true; + lastReceivedOffer.sender = mp.from; + lastReceivedOffer.has_channel = b->has_offer_channel; + if (b->has_offer_channel) + lastReceivedOffer.channel = b->offer_channel; + lastReceivedOffer.region = b->offer_region; + lastReceivedOffer.preset = b->offer_preset; + lastReceivedOffer.received_at = + getValidTime(RTCQualityFromNet); // 0 if no RTC fix yet — consumers must not treat 0 as valid + LOG_INFO("Beacon: stored offer from 0x%08x (preset=%d)", mp.from, b->offer_preset); + } + + notifyObservers(&mp); + return false; +} diff --git a/src/modules/MeshBeaconModule.h b/src/modules/MeshBeaconModule.h new file mode 100644 index 000000000..5baea179b --- /dev/null +++ b/src/modules/MeshBeaconModule.h @@ -0,0 +1,163 @@ +#pragma once +#include "MeshRadio.h" +#include "Observer.h" +#include "ProtobufModule.h" +#include "RadioInterface.h" +#include "concurrency/OSThread.h" +#include "mesh/generated/meshtastic/mesh_beacon.pb.h" +#include "mesh/generated/meshtastic/module_config.pb.h" + +// Short aliases for the MeshBeaconConfig.flags bitfield (see module_config.proto MeshBeaconConfig.Flags). +#define MESH_BEACON_FLAG_LISTEN_ENABLED meshtastic_ModuleConfig_MeshBeaconConfig_Flags_FLAG_LISTEN_ENABLED +#define MESH_BEACON_FLAG_BROADCAST_ENABLED meshtastic_ModuleConfig_MeshBeaconConfig_Flags_FLAG_BROADCAST_ENABLED +#define MESH_BEACON_FLAG_LEGACY_SPLIT meshtastic_ModuleConfig_MeshBeaconConfig_Flags_FLAG_LEGACY_SPLIT + +// Sidecar entry pairing a packet ID with target radio settings for beacon TX. +typedef struct { + bool inUse; + PacketId id; + meshtastic_Config_LoRaConfig_ModemPreset preset; + uint16_t slot; + // When true, reconfigureForBeaconTX sets hop_start=1 so pre-2.7.20 firmware + // (which drops hop_start==0 packets) accepts the zero-hop beacon. + bool legacyHopOverride; + // Per-target radio settings. UNSET region means use current lora.region. + meshtastic_Config_LoRaConfig_RegionCode region; + bool has_channel; + meshtastic_ChannelSettings channel; +} MeshBeaconModule_TargetRadioSettings; + +/** + * Base class: holds the radio-switching sidecar table and static helpers. + * The sidecar avoids touching MeshPacket proto fields for per-packet radio state. + */ +class MeshBeaconModule +{ + public: + MeshBeaconModule(); + + /** + * Reconfigure the radio for beacon TX, or restore to original config if p is NULL. + * Returns true if the radio was reconfigured (caller must re-run transmit delay for CCA). + * Driven by broadcast_on_preset / broadcast_on_channel from MeshBeaconConfig. + */ + static bool reconfigureForBeaconTX(RadioInterface *iface, meshtastic_MeshPacket *p); + + /** + * Associate target radio settings with an outgoing packet by its ID. + * Sidecar holds 8 entries; evicts slot 0 on overflow. + */ + static void + setTargetRadioSettings(const meshtastic_MeshPacket *p, meshtastic_Config_LoRaConfig_ModemPreset preset, uint16_t slot, + bool legacyHopOverride = false, + meshtastic_Config_LoRaConfig_RegionCode region = meshtastic_Config_LoRaConfig_RegionCode_UNSET, + bool has_channel = false, const meshtastic_ChannelSettings *channel = nullptr); + + /** + * Returns true if the sidecar table contains an entry for this packet's ID. + * Used by RadioLibInterface to gate the channel-active check. + */ + static bool hasTargetRadioSettings(const meshtastic_MeshPacket *p); + + /** + * Remove the sidecar entry for this packet after it has been sent. + * Called from RadioLibInterface::completeSending(). + */ + static void clearTargetRadioSettings(const meshtastic_MeshPacket *p); + + /** + * True if p is tagged for a beacon radio switch whose target config must NOT be transmitted: + * preset invalid for the target region, or an unlicensed node would key up on a ham-only + * (licensed-only) region. The radio driver drops such packets rather than sending them on the + * current config. False for any packet without a sidecar entry (normal traffic is never affected). + */ + static bool beaconTxConfigInvalid(const meshtastic_MeshPacket *p); + + protected: + /** + * Build the ChannelSettings the beacon transmits on: the base (primary) channel overlaid with + * any broadcast_on_channel overrides, defaulting an empty name to the target preset's display + * name. Shared by the encrypt-time channel swap and the radio-thread RF swap so the channel + * key + hash are identical at both points. + */ + static meshtastic_ChannelSettings beaconChannelSettings(const meshtastic_ChannelSettings &base, + meshtastic_Config_LoRaConfig_ModemPreset preset, + const meshtastic_ChannelSettings *overrideChannel = nullptr); + + static meshtastic_Config_LoRaConfig_ModemPreset originalModemPreset; + static uint16_t originalLoraChannel; + static meshtastic_Config_LoRaConfig_RegionCode originalRegion; + static meshtastic_ChannelSettings originalPrimaryChannel; +}; + +/** + * Broadcaster: periodically sends MeshBeacon packets on the configured preset/channel. + * Active only when the FLAG_BROADCAST_ENABLED bit is set in moduleConfig.mesh_beacon.flags. + * Inherits ProtobufModule to access allocDataProtobuf + setStartDelay. + * + * Packet flow: + * Normal (combined): one MESH_BEACON_APP carrying offer + message on the beacon radio config. + * Legacy split: two packets when both text and offer are present and FLAG_LEGACY_SPLIT is set, + * both sent on the same beacon radio settings: + * A) MESH_BEACON_APP with offer only (no text). + * B) TEXT_MESSAGE_APP with the text (for clients that only decode TEXT_MESSAGE_APP). + */ +class MeshBeaconBroadcastModule : private MeshBeaconModule, + public ProtobufModule, + private concurrency::OSThread +{ + public: + MeshBeaconBroadcastModule(); + + // Mark the cached payload dirty (call after config change). + void invalidateCache() { payloadCacheDirty = true; } + + protected: + virtual bool handleReceivedProtobuf(const meshtastic_MeshPacket &, meshtastic_MeshBeacon *) override { return false; } + virtual int32_t runOnce() override; + + protected: + void sendBeacon(); + void rebuildCache(); + + // Send one beacon packet. When overrideChannel is set and has a name/PSK override, + // the packet is encrypted with that channel's key (not the primary's). + void sendBeaconPacket(meshtastic_MeshPacket *p, meshtastic_Config_LoRaConfig_ModemPreset targetPreset, + bool has_channel = false, const meshtastic_ChannelSettings *overrideChannel = nullptr); + + bool payloadCacheDirty = true; + uint8_t payloadCache[meshtastic_MeshBeacon_size] = {}; + pb_size_t payloadCacheSize = 0; +}; +extern MeshBeaconBroadcastModule *meshBeaconBroadcastModule; + +/** + * Listener: receives MESH_BEACON_APP packets and caches any offered channel/preset for the client + * app to retrieve. It does NOT unwrap the text into a separate message — the original beacon packet + * already reaches the client (handler returns CONTINUE), which reads `message` from it directly. + * Does NOT auto-apply offered settings — client app must do so explicitly. + * Active only when the FLAG_LISTEN_ENABLED bit is set in moduleConfig.mesh_beacon.flags. + */ +class MeshBeaconListenerModule : public ProtobufModule, public Observable +{ + public: + MeshBeaconListenerModule(); + + struct BeaconOffer { + bool valid; + NodeNum sender; + bool has_channel; + meshtastic_ChannelSettings channel; + meshtastic_Config_LoRaConfig_RegionCode region; + meshtastic_Config_LoRaConfig_ModemPreset preset; + uint32_t received_at; + }; + + // Last received offer — accessible to admin/API for client app retrieval. + static BeaconOffer lastReceivedOffer; + + protected: + virtual bool handleReceivedProtobuf(const meshtastic_MeshPacket &mp, meshtastic_MeshBeacon *b) override; + virtual bool wantPacket(const meshtastic_MeshPacket *p) override; +}; +extern MeshBeaconListenerModule *meshBeaconListenerModule; diff --git a/src/modules/Modules.cpp b/src/modules/Modules.cpp index 89c5b9d43..1e9380575 100644 --- a/src/modules/Modules.cpp +++ b/src/modules/Modules.cpp @@ -28,6 +28,9 @@ #if !MESHTASTIC_EXCLUDE_NODEINFO #include "modules/NodeInfoModule.h" #endif +#if !MESHTASTIC_EXCLUDE_BEACON +#include "modules/MeshBeaconModule.h" +#endif #if !MESHTASTIC_EXCLUDE_GPS #include "modules/PositionModule.h" #endif @@ -143,6 +146,10 @@ void setupModules() #if !MESHTASTIC_EXCLUDE_NODEINFO nodeInfoModule = new NodeInfoModule(); #endif +#if !MESHTASTIC_EXCLUDE_BEACON + meshBeaconBroadcastModule = new MeshBeaconBroadcastModule(); + meshBeaconListenerModule = new MeshBeaconListenerModule(); +#endif #if !MESHTASTIC_EXCLUDE_GPS positionModule = new PositionModule(); #endif diff --git a/test/native-suite-count b/test/native-suite-count index a45fd52cc..7273c0fa8 100644 --- a/test/native-suite-count +++ b/test/native-suite-count @@ -1 +1 @@ -24 +25 diff --git a/test/test_mesh_beacon/test_main.cpp b/test/test_mesh_beacon/test_main.cpp new file mode 100644 index 000000000..f19fac087 --- /dev/null +++ b/test/test_mesh_beacon/test_main.cpp @@ -0,0 +1,1457 @@ +/** + * Unit tests for MeshBeaconModule: + * - AdminModule::handleSetModuleConfig validation (invalid/valid inputs) + * - MeshBeaconBroadcastModule payload cache lifecycle + * - MeshBeaconBroadcastModule::sendBeacon sends a correctly formed packet + * - MeshBeaconListenerModule offer caching and empty-message guard + */ + +#include "TestUtil.h" +#include + +#if defined(ARCH_PORTDUINO) +#define BEACON_TEST_ENTRY extern "C" +#else +#define BEACON_TEST_ENTRY +#endif + +#if !MESHTASTIC_EXCLUDE_BEACON + +#include "MeshRadio.h" +#include "MeshService.h" +#include "NodeDB.h" +#include "RadioInterface.h" +#include "airtime.h" +#include "modules/AdminModule.h" +#include "modules/MeshBeaconModule.h" +#include +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// Formatted diagnostic helper. TEST_MESSAGE emits a line into Unity's per-test +// output (shown inline alongside the :PASS/:FAIL result); use this when you need +// a printf-style formatted note tied to a specific assertion. Plain printf() also +// works for free-standing log lines (e.g. the group headers in setup() below). +// --------------------------------------------------------------------------- +#define MSG_BUF_LEN 256 +#define TEST_MSG_FMT(fmt, ...) \ + do { \ + char _buf[MSG_BUF_LEN]; \ + snprintf(_buf, sizeof(_buf), fmt, ##__VA_ARGS__); \ + TEST_MESSAGE(_buf); \ + } while (0) + +namespace +{ + +constexpr NodeNum kLocalNode = 0xAAAA0001; +constexpr NodeNum kRemoteNode = 0xBBBB0002; + +// --------------------------------------------------------------------------- +// Minimal MockMeshService — stubs out side-effecting virtuals. +// handleToRadio is non-virtual so it runs the real implementation; we guard +// against the router->sendLocal path by setting a MockRouter below. +// --------------------------------------------------------------------------- +class MockMeshService : public MeshService +{ + public: + void sendClientNotification(meshtastic_ClientNotification *n) override { releaseClientNotificationToPool(n); } +}; + +// --------------------------------------------------------------------------- +// MockRouter: captures every packet handed to send() instead of transmitting. +// --------------------------------------------------------------------------- +class MockRouter : public Router +{ + public: + ~MockRouter() + { + delete cryptLock; + cryptLock = nullptr; + } + + ErrorCode send(meshtastic_MeshPacket *p) override + { + // Capture the primary channel as seen AT send() time. sendBeaconPacket() temporarily swaps + // the beacon channel into the primary slot around this call so perhapsEncode keys off it, + // so this snapshot reflects which channel the packet would actually be encrypted on. + if (channelFile.channels_count > 0) + primaryAtSend.push_back(channels.getByIndex(channels.getPrimaryIndex()).settings); + sentPackets.push_back(*p); + packetPool.release(p); + return ERRNO_OK; + } + + // Locally-addressed packets land here instead of send(). Release immediately + // rather than queuing into fromRadioQueue (which is never drained in tests). + void enqueueReceivedMessage(meshtastic_MeshPacket *p) override { packetPool.release(p); } + + std::vector sentPackets; + std::vector primaryAtSend; +}; + +// --------------------------------------------------------------------------- +// AdminModuleTestShim — exposes protected handleSetModuleConfig. +// --------------------------------------------------------------------------- +class AdminModuleTestShim : public AdminModule +{ + public: + using AdminModule::handleSetModuleConfig; +}; + +// --------------------------------------------------------------------------- +// MeshBeaconBroadcastModuleTestShim — exposes private internals for testing. +// --------------------------------------------------------------------------- +class MeshBeaconBroadcastModuleTestShim : public MeshBeaconBroadcastModule +{ + public: + using MeshBeaconBroadcastModule::payloadCache; + using MeshBeaconBroadcastModule::payloadCacheDirty; + using MeshBeaconBroadcastModule::payloadCacheSize; + using MeshBeaconBroadcastModule::rebuildCache; + using MeshBeaconBroadcastModule::runOnce; + using MeshBeaconBroadcastModule::sendBeacon; +}; + +// --------------------------------------------------------------------------- +// MeshBeaconListenerModuleTestShim — exposes handleReceivedProtobuf. +// --------------------------------------------------------------------------- +class MeshBeaconListenerModuleTestShim : public MeshBeaconListenerModule +{ + public: + using MeshBeaconListenerModule::handleReceivedProtobuf; + using MeshBeaconListenerModule::wantPacket; +}; + +// --------------------------------------------------------------------------- +// Globals managed by setUp / tearDown. +// --------------------------------------------------------------------------- +static MockMeshService *mockSvc = nullptr; +static MockRouter *mockRouter = nullptr; +static AdminModuleTestShim *testAdmin = nullptr; +static AirTime *testAirTime = nullptr; + +// --------------------------------------------------------------------------- +// Helper: build a ModuleConfig wrapper for the beacon case (mirrors the wire +// format used by set_module_config admin messages). +// --------------------------------------------------------------------------- +static meshtastic_ModuleConfig makeBeaconModuleConfig(meshtastic_ModuleConfig_MeshBeaconConfig bcfg) +{ + meshtastic_ModuleConfig mc = meshtastic_ModuleConfig_init_zero; + mc.which_payload_variant = meshtastic_ModuleConfig_mesh_beacon_tag; + mc.payload_variant.mesh_beacon = bcfg; + return mc; +} + +// --------------------------------------------------------------------------- +// Helper: reset module/device config to a known baseline. +// --------------------------------------------------------------------------- +static void resetConfig() +{ + moduleConfig = meshtastic_LocalModuleConfig_init_zero; + config = meshtastic_LocalConfig_init_zero; + + // Device is an EU_868 node with LONG_FAST — the starting point. + config.lora.region = meshtastic_Config_LoRaConfig_RegionCode_EU_868; + config.lora.use_preset = true; + config.lora.modem_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_FAST; + // Allow TX unconditionally so airtime checks don't block sendBeacon(). + config.lora.override_duty_cycle = true; + config.device.role = meshtastic_Config_DeviceConfig_Role_CLIENT; + + myNodeInfo.my_node_num = kLocalNode; + + initRegion(); +} + +// Install a single PRIMARY channel with an explicit name + PSK so the beacon channel-swap path +// (sendBeaconPacket) has a real primary slot to save/restore and to read the active PSK from. +static void installTestPrimaryChannel(const char *name, const uint8_t *psk, size_t pskLen) +{ + channelFile.channels_count = 1; + meshtastic_Channel &ch = channelFile.channels[0]; + ch = meshtastic_Channel_init_zero; + ch.index = 0; + ch.has_settings = true; + ch.role = meshtastic_Channel_Role_PRIMARY; + strncpy(ch.settings.name, name, sizeof(ch.settings.name) - 1); + ch.settings.psk.size = (pb_size_t)pskLen; + memcpy(ch.settings.psk.bytes, psk, pskLen); + channels.onConfigChanged(); // set primaryIndex + recompute hashes +} + +// Install a secondary channel at the given table index. Pass psk=nullptr/pskLen=0 for a blank slot +// (no name, no PSK) to exercise the "referenced slot is unconfigured" fallback. +static void installTestSecondaryChannel(uint8_t index, const char *name, const uint8_t *psk, size_t pskLen) +{ + if (channelFile.channels_count < (pb_size_t)(index + 1)) + channelFile.channels_count = index + 1; + meshtastic_Channel &ch = channelFile.channels[index]; + ch = meshtastic_Channel_init_zero; + ch.index = index; + ch.has_settings = true; + ch.role = meshtastic_Channel_Role_SECONDARY; + if (name) + strncpy(ch.settings.name, name, sizeof(ch.settings.name) - 1); + if (psk && pskLen) { + ch.settings.psk.size = (pb_size_t)pskLen; + memcpy(ch.settings.psk.bytes, psk, pskLen); + } + channels.onConfigChanged(); +} + +// =========================================================================== +// Group 1: AdminModule config validation — bad inputs must be sanitised +// =========================================================================== + +/** + * Verify SHORT_TURBO is rejected when the region is EU_868 (turbo presets are not in that + * region's allowed preset set). Important to catch regressions where admin stores unlawful + * radio settings that would violate regional radio regulations. + */ +static void test_adminValidation_turboPresetOnEU868_isCleared(void) +{ + resetConfig(); + + meshtastic_ModuleConfig_MeshBeaconConfig bcfg = meshtastic_ModuleConfig_MeshBeaconConfig_init_zero; + bcfg.flags |= MESH_BEACON_FLAG_BROADCAST_ENABLED; + bcfg.has_broadcast_on_preset = true; + bcfg.broadcast_on_preset = meshtastic_Config_LoRaConfig_ModemPreset_SHORT_TURBO; + + testAdmin->handleSetModuleConfig(makeBeaconModuleConfig(bcfg)); + + TEST_ASSERT_TRUE(moduleConfig.has_mesh_beacon); + TEST_ASSERT_FALSE_MESSAGE(moduleConfig.mesh_beacon.has_broadcast_on_preset, "SHORT_TURBO must be cleared for EU_868"); +} + +/** + * Verify LONG_TURBO is also cleared for EU_868, not just SHORT_TURBO. + * Important to confirm rejection covers the entire turbo preset family rather than one variant. + */ +static void test_adminValidation_longTurboPresetOnEU868_isCleared(void) +{ + resetConfig(); + + meshtastic_ModuleConfig_MeshBeaconConfig bcfg = meshtastic_ModuleConfig_MeshBeaconConfig_init_zero; + bcfg.has_broadcast_on_preset = true; + bcfg.broadcast_on_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_TURBO; + + testAdmin->handleSetModuleConfig(makeBeaconModuleConfig(bcfg)); + + TEST_ASSERT_FALSE(moduleConfig.mesh_beacon.has_broadcast_on_preset); +} + +/** + * Verify a turbo preset passes validation for US (PROFILE_STD allows all presets). + * Important because the same preset that is illegal in EU_868 must be preserved in permissive regions. + */ +static void test_adminValidation_turboPresetOnUS_isAccepted(void) +{ + resetConfig(); + config.lora.region = meshtastic_Config_LoRaConfig_RegionCode_US; + initRegion(); + + meshtastic_ModuleConfig_MeshBeaconConfig bcfg = meshtastic_ModuleConfig_MeshBeaconConfig_init_zero; + bcfg.has_broadcast_on_preset = true; + bcfg.broadcast_on_preset = meshtastic_Config_LoRaConfig_ModemPreset_SHORT_TURBO; + + testAdmin->handleSetModuleConfig(makeBeaconModuleConfig(bcfg)); + + TEST_ASSERT_TRUE(moduleConfig.mesh_beacon.has_broadcast_on_preset); + TEST_ASSERT_EQUAL(meshtastic_Config_LoRaConfig_ModemPreset_SHORT_TURBO, moduleConfig.mesh_beacon.broadcast_on_preset); +} + +/** + * Verify an out-of-range region code (255) is sanitised to UNSET rather than stored verbatim. + * Important to prevent invalid proto enum values from reaching the broadcaster and being broadcast + * over the air. + */ +static void test_adminValidation_unknownOfferRegion_isCleared(void) +{ + resetConfig(); + + meshtastic_ModuleConfig_MeshBeaconConfig bcfg = meshtastic_ModuleConfig_MeshBeaconConfig_init_zero; + bcfg.broadcast_offer_region = (meshtastic_Config_LoRaConfig_RegionCode)255; + + testAdmin->handleSetModuleConfig(makeBeaconModuleConfig(bcfg)); + + TEST_ASSERT_EQUAL(meshtastic_Config_LoRaConfig_RegionCode_UNSET, moduleConfig.mesh_beacon.broadcast_offer_region); +} + +/** + * Verify a known-good offer region (US) is written through unchanged after admin validation. + * Important as a positive-path control alongside the rejection tests. + */ +static void test_adminValidation_validOfferRegion_isPreserved(void) +{ + resetConfig(); + + meshtastic_ModuleConfig_MeshBeaconConfig bcfg = meshtastic_ModuleConfig_MeshBeaconConfig_init_zero; + bcfg.broadcast_offer_region = meshtastic_Config_LoRaConfig_RegionCode_US; + + testAdmin->handleSetModuleConfig(makeBeaconModuleConfig(bcfg)); + + TEST_ASSERT_EQUAL(meshtastic_Config_LoRaConfig_RegionCode_US, moduleConfig.mesh_beacon.broadcast_offer_region); +} + +/** + * Verify an out-of-range region in a multi-target entry is sanitised to UNSET on write. + * Important because broadcast_targets entries are validated independently of the single-target + * broadcast_on_* fields, and an invalid enum must never reach the radio-switch path. + */ +static void test_adminValidation_targetUnknownRegion_isCleared(void) +{ + resetConfig(); + + meshtastic_ModuleConfig_MeshBeaconConfig bcfg = meshtastic_ModuleConfig_MeshBeaconConfig_init_zero; + bcfg.broadcast_targets_count = 1; + bcfg.broadcast_targets[0].region = (meshtastic_Config_LoRaConfig_RegionCode)255; + + testAdmin->handleSetModuleConfig(makeBeaconModuleConfig(bcfg)); + + TEST_ASSERT_EQUAL(1, moduleConfig.mesh_beacon.broadcast_targets_count); + TEST_ASSERT_EQUAL(meshtastic_Config_LoRaConfig_RegionCode_UNSET, moduleConfig.mesh_beacon.broadcast_targets[0].region); +} + +/** + * Verify a preset that is illegal for a multi-target entry's region clears that entry's preset + * (and its channel), matching the single-target broadcast_on_preset rule. + */ +static void test_adminValidation_targetInvalidPresetForRegion_isCleared(void) +{ + resetConfig(); + + meshtastic_ModuleConfig_MeshBeaconConfig bcfg = meshtastic_ModuleConfig_MeshBeaconConfig_init_zero; + bcfg.broadcast_targets_count = 1; + bcfg.broadcast_targets[0].region = meshtastic_Config_LoRaConfig_RegionCode_EU_868; + bcfg.broadcast_targets[0].has_preset = true; + bcfg.broadcast_targets[0].preset = meshtastic_Config_LoRaConfig_ModemPreset_SHORT_TURBO; + bcfg.broadcast_targets[0].has_channel_index = true; + bcfg.broadcast_targets[0].channel_index = 1; + + testAdmin->handleSetModuleConfig(makeBeaconModuleConfig(bcfg)); + + TEST_ASSERT_FALSE_MESSAGE(moduleConfig.mesh_beacon.broadcast_targets[0].has_preset, + "SHORT_TURBO must be cleared for EU_868 target"); + TEST_ASSERT_FALSE(moduleConfig.mesh_beacon.broadcast_targets[0].has_channel_index); +} + +/** + * Verify a channel_index beyond the channel-table capacity is cleared on write. + * Important so the broadcaster never indexes out of bounds when resolving a target channel. + */ +static void test_adminValidation_targetChannelIndexOutOfRange_isCleared(void) +{ + resetConfig(); + + meshtastic_ModuleConfig_MeshBeaconConfig bcfg = meshtastic_ModuleConfig_MeshBeaconConfig_init_zero; + bcfg.broadcast_targets_count = 1; + bcfg.broadcast_targets[0].has_channel_index = true; + bcfg.broadcast_targets[0].channel_index = MAX_NUM_CHANNELS; // one past the last valid slot + + testAdmin->handleSetModuleConfig(makeBeaconModuleConfig(bcfg)); + + TEST_ASSERT_FALSE(moduleConfig.mesh_beacon.broadcast_targets[0].has_channel_index); +} + +/** + * Verify an in-range channel_index survives admin validation unchanged. + */ +static void test_adminValidation_targetChannelIndexInRange_isPreserved(void) +{ + resetConfig(); + + meshtastic_ModuleConfig_MeshBeaconConfig bcfg = meshtastic_ModuleConfig_MeshBeaconConfig_init_zero; + bcfg.broadcast_targets_count = 1; + bcfg.broadcast_targets[0].has_channel_index = true; + bcfg.broadcast_targets[0].channel_index = 0; + + testAdmin->handleSetModuleConfig(makeBeaconModuleConfig(bcfg)); + + TEST_ASSERT_TRUE(moduleConfig.mesh_beacon.broadcast_targets[0].has_channel_index); + TEST_ASSERT_EQUAL_UINT32(0, moduleConfig.mesh_beacon.broadcast_targets[0].channel_index); +} + +/** + * Verify a valid preset/region multi-target entry survives admin validation unchanged. + * Positive-path control for the per-target validation. + */ +static void test_adminValidation_targetValidPresetForRegion_isPreserved(void) +{ + resetConfig(); + config.lora.region = meshtastic_Config_LoRaConfig_RegionCode_US; + initRegion(); + + meshtastic_ModuleConfig_MeshBeaconConfig bcfg = meshtastic_ModuleConfig_MeshBeaconConfig_init_zero; + bcfg.broadcast_targets_count = 1; + bcfg.broadcast_targets[0].region = meshtastic_Config_LoRaConfig_RegionCode_US; + bcfg.broadcast_targets[0].has_preset = true; + bcfg.broadcast_targets[0].preset = meshtastic_Config_LoRaConfig_ModemPreset_SHORT_TURBO; + + testAdmin->handleSetModuleConfig(makeBeaconModuleConfig(bcfg)); + + TEST_ASSERT_TRUE(moduleConfig.mesh_beacon.broadcast_targets[0].has_preset); + TEST_ASSERT_EQUAL(meshtastic_Config_LoRaConfig_ModemPreset_SHORT_TURBO, moduleConfig.mesh_beacon.broadcast_targets[0].preset); + TEST_ASSERT_EQUAL(meshtastic_Config_LoRaConfig_RegionCode_US, moduleConfig.mesh_beacon.broadcast_targets[0].region); +} + +/** + * Verify broadcast_message is hard-capped at 100 characters (NUL forced at index 100). + * Important to prevent oversized beacon payloads from abusing airtime across the mesh. + */ +static void test_adminValidation_messageTooLong_isTruncatedAt100(void) +{ + resetConfig(); + + meshtastic_ModuleConfig_MeshBeaconConfig bcfg = meshtastic_ModuleConfig_MeshBeaconConfig_init_zero; + // Fill with 'A' up to the full array size; admin must enforce ≤100 chars. + memset(bcfg.broadcast_message, 'A', sizeof(bcfg.broadcast_message)); + bcfg.broadcast_message[sizeof(bcfg.broadcast_message) - 1] = '\0'; // pb_decode guarantee + + testAdmin->handleSetModuleConfig(makeBeaconModuleConfig(bcfg)); + + // Byte at index 100 must be NUL (length capped at 100). + TEST_ASSERT_EQUAL('\0', moduleConfig.mesh_beacon.broadcast_message[100]); + // Bytes before it should still be 'A'. + TEST_ASSERT_EQUAL('A', moduleConfig.mesh_beacon.broadcast_message[0]); +} + +/** + * Verify any non-zero interval below 3600 s is clamped up to the 1-hour minimum. + * Important to prevent high-rate beacon floods from a misconfigured or malicious client. + */ +static void test_adminValidation_intervalTooLow_isClamped(void) +{ + resetConfig(); + + meshtastic_ModuleConfig_MeshBeaconConfig bcfg = meshtastic_ModuleConfig_MeshBeaconConfig_init_zero; + bcfg.broadcast_interval_secs = 60; // way below minimum + + testAdmin->handleSetModuleConfig(makeBeaconModuleConfig(bcfg)); + + TEST_ASSERT_EQUAL_UINT32(3600, moduleConfig.mesh_beacon.broadcast_interval_secs); +} + +/** + * Verify an interval above the minimum is stored as-is without modification. + * Important to confirm the clamp is one-sided (lower bound only, no upper bound enforced). + */ +static void test_adminValidation_intervalTooHigh_isPreserved(void) +{ + resetConfig(); + + meshtastic_ModuleConfig_MeshBeaconConfig bcfg = meshtastic_ModuleConfig_MeshBeaconConfig_init_zero; + bcfg.broadcast_interval_secs = 999999; + + testAdmin->handleSetModuleConfig(makeBeaconModuleConfig(bcfg)); + + TEST_ASSERT_EQUAL_UINT32(999999, moduleConfig.mesh_beacon.broadcast_interval_secs); +} + +/** + * Verify LONG_FAST (enum value 0) survives admin validation without being treated as 'absent'. + * Important to guard the has_broadcast_offer_preset presence-flag fix against zero-value erasure. + */ +static void test_adminValidation_longFastOfferPreset_isPreserved(void) +{ + resetConfig(); + + meshtastic_ModuleConfig_MeshBeaconConfig bcfg = meshtastic_ModuleConfig_MeshBeaconConfig_init_zero; + bcfg.has_broadcast_offer_preset = true; + bcfg.broadcast_offer_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_FAST; + + testAdmin->handleSetModuleConfig(makeBeaconModuleConfig(bcfg)); + + TEST_ASSERT_TRUE(moduleConfig.mesh_beacon.has_broadcast_offer_preset); + TEST_ASSERT_EQUAL(meshtastic_Config_LoRaConfig_ModemPreset_LONG_FAST, moduleConfig.mesh_beacon.broadcast_offer_preset); +} + +/** + * Verify that interval 0 (the documented 'use default' sentinel) is not raised to 3600 by the clamp. + * Important because 0 and 3600 have different runtime semantics in runOnce via + * Default::getConfiguredOrDefault. + */ +static void test_adminValidation_intervalZero_isNotClamped(void) +{ + resetConfig(); + + meshtastic_ModuleConfig_MeshBeaconConfig bcfg = meshtastic_ModuleConfig_MeshBeaconConfig_init_zero; + bcfg.broadcast_interval_secs = 0; + + testAdmin->handleSetModuleConfig(makeBeaconModuleConfig(bcfg)); + + TEST_ASSERT_EQUAL_UINT32(0, moduleConfig.mesh_beacon.broadcast_interval_secs); +} + +/** + * Verify that saving a new beacon config marks the broadcaster's payload cache dirty. + * Important so the next TX re-encodes from the latest config rather than a pre-save stale snapshot. + */ +static void test_adminValidation_validSave_invalidatesCache(void) +{ + resetConfig(); + + // Prime the broadcaster with a clean state so the dirty flag is known. + std::unique_ptr bcast(new MeshBeaconBroadcastModuleTestShim()); + meshBeaconBroadcastModule = bcast.get(); + bcast->payloadCacheDirty = false; // pretend it was freshly built + + meshtastic_ModuleConfig_MeshBeaconConfig bcfg = meshtastic_ModuleConfig_MeshBeaconConfig_init_zero; + bcfg.flags |= MESH_BEACON_FLAG_BROADCAST_ENABLED; + strncpy(bcfg.broadcast_message, "hello", sizeof(bcfg.broadcast_message) - 1); + + testAdmin->handleSetModuleConfig(makeBeaconModuleConfig(bcfg)); + + TEST_ASSERT_TRUE_MESSAGE(bcast->payloadCacheDirty, "Config save must mark payload cache dirty"); + + meshBeaconBroadcastModule = nullptr; +} + +// =========================================================================== +// Group 2: Broadcaster payload cache +// =========================================================================== + +/** + * Verify rebuildCache produces at least one encoded byte when broadcast_message is set. + * Important as the most basic liveness check for the protobuf encoding path. + */ +static void test_broadcaster_rebuildCache_producesNonEmptyPayload(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + strncpy(moduleConfig.mesh_beacon.broadcast_message, "Test beacon", sizeof(moduleConfig.mesh_beacon.broadcast_message) - 1); + + MeshBeaconBroadcastModuleTestShim bcast; + TEST_ASSERT_TRUE(bcast.payloadCacheDirty); + + bcast.rebuildCache(); + + TEST_ASSERT_FALSE_MESSAGE(bcast.payloadCacheDirty, "rebuildCache must clear dirty flag"); + TEST_ASSERT_GREATER_THAN_MESSAGE(0, (int)bcast.payloadCacheSize, "rebuildCache must produce a non-empty payload"); +} + +/** + * Verify the cached bytes round-trip through pb_decode back to the original message string. + * Important to catch any protobuf field-tag or wire-type regression in the encoding path. + */ +static void test_broadcaster_rebuildCache_payloadDecodesCorrectly(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + const char *msg = "Hello, Meshtastic!"; + strncpy(moduleConfig.mesh_beacon.broadcast_message, msg, sizeof(moduleConfig.mesh_beacon.broadcast_message) - 1); + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.rebuildCache(); + + // Decode the cached bytes back into a MeshBeacon struct. + meshtastic_MeshBeacon decoded = meshtastic_MeshBeacon_init_zero; + pb_istream_t stream = pb_istream_from_buffer(bcast.payloadCache, bcast.payloadCacheSize); + bool ok = pb_decode(&stream, &meshtastic_MeshBeacon_msg, &decoded); + + TEST_ASSERT_TRUE_MESSAGE(ok, "Cached payload must decode without error"); + TEST_ASSERT_EQUAL_STRING(msg, decoded.message); +} + +/** + * Verify offer_region and offer_preset are present in the encoded cache payload. + * Important to confirm the offer-carrying path correctly uses the has_broadcast_offer_preset flag. + */ +static void test_broadcaster_rebuildCache_offerFieldsEncoded(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.broadcast_offer_region = meshtastic_Config_LoRaConfig_RegionCode_US; + moduleConfig.mesh_beacon.has_broadcast_offer_preset = true; + moduleConfig.mesh_beacon.broadcast_offer_preset = meshtastic_Config_LoRaConfig_ModemPreset_MEDIUM_FAST; + strncpy(moduleConfig.mesh_beacon.broadcast_message, "offer-test", sizeof(moduleConfig.mesh_beacon.broadcast_message) - 1); + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.rebuildCache(); + + meshtastic_MeshBeacon decoded = meshtastic_MeshBeacon_init_zero; + pb_istream_t stream = pb_istream_from_buffer(bcast.payloadCache, bcast.payloadCacheSize); + pb_decode(&stream, &meshtastic_MeshBeacon_msg, &decoded); + + TEST_ASSERT_EQUAL(meshtastic_Config_LoRaConfig_RegionCode_US, decoded.offer_region); + TEST_ASSERT_EQUAL(meshtastic_Config_LoRaConfig_ModemPreset_MEDIUM_FAST, decoded.offer_preset); +} + +/** + * Verify invalidateCache flips payloadCacheDirty back to true after a successful rebuild. + * Important to confirm the cache-invalidation contract relied on by admin saves and config observers. + */ +static void test_broadcaster_invalidateCache_setsDirtyFlag(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.rebuildCache(); + TEST_ASSERT_FALSE(bcast.payloadCacheDirty); + + bcast.invalidateCache(); + TEST_ASSERT_TRUE(bcast.payloadCacheDirty); +} + +/** + * Verify calling rebuildCache a second time without an intervening invalidation is a no-op. + * Important to prevent spurious re-encodes when config-observer callbacks fire multiple times. + */ +static void test_broadcaster_rebuildCache_idempotent(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + strncpy(moduleConfig.mesh_beacon.broadcast_message, "idem", sizeof(moduleConfig.mesh_beacon.broadcast_message) - 1); + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.rebuildCache(); + pb_size_t firstSize = bcast.payloadCacheSize; + bcast.rebuildCache(); // second call — should be identical + pb_size_t secondSize = bcast.payloadCacheSize; + + TEST_ASSERT_FALSE(bcast.payloadCacheDirty); + TEST_ASSERT_EQUAL(firstSize, secondSize); +} + +// =========================================================================== +// Group 3: Broadcaster sendBeacon — packet structure +// =========================================================================== + +/** + * Verify the 'from' field defaults to the local node number when broadcast_send_as_node is 0. + * Important for correct source attribution in peer node tables that receive the beacon. + */ +static void test_broadcaster_sendBeacon_fromIsLocalNodeWhenUnset(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.flags |= MESH_BEACON_FLAG_BROADCAST_ENABLED; + moduleConfig.mesh_beacon.broadcast_send_as_node = 0; + strncpy(moduleConfig.mesh_beacon.broadcast_message, "from-local", sizeof(moduleConfig.mesh_beacon.broadcast_message) - 1); + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.sendBeacon(); + + TEST_ASSERT_EQUAL_UINT32(1, mockRouter->sentPackets.size()); + TEST_ASSERT_EQUAL_UINT32(kLocalNode, mockRouter->sentPackets[0].from); +} + +/** + * Verify broadcast_send_as_node is currently disabled: 'from' is always the local node + * even when broadcast_send_as_node is set to a remote node number. + * (broadcast_send_as_node is commented out as "not suitable right now".) + */ +static void test_broadcaster_sendBeacon_fromIsCustomNodeWhenSet(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.flags |= MESH_BEACON_FLAG_BROADCAST_ENABLED; + moduleConfig.mesh_beacon.broadcast_send_as_node = kRemoteNode; + strncpy(moduleConfig.mesh_beacon.broadcast_message, "from-remote", sizeof(moduleConfig.mesh_beacon.broadcast_message) - 1); + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.sendBeacon(); + + TEST_ASSERT_EQUAL_UINT32(1, mockRouter->sentPackets.size()); + // broadcast_send_as_node is disabled; from is always the local node + TEST_ASSERT_EQUAL_UINT32(kLocalNode, mockRouter->sentPackets[0].from); +} + +/** + * Verify the 'to' field is always NODENUM_BROADCAST regardless of other settings. + * Important because beacons are mesh-wide announcements and must never be addressed to a single peer. + */ +static void test_broadcaster_sendBeacon_addressedToBroadcast(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + strncpy(moduleConfig.mesh_beacon.broadcast_message, "bcast-addr", sizeof(moduleConfig.mesh_beacon.broadcast_message) - 1); + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.sendBeacon(); + + TEST_ASSERT_EQUAL_UINT32(1, mockRouter->sentPackets.size()); + TEST_ASSERT_EQUAL_UINT32(NODENUM_BROADCAST, mockRouter->sentPackets[0].to); +} + +/** + * Verify MESH_BEACON_APP portnum is used when the packet carries a radio offer payload. + * Important so receivers use the structured protobuf decoder rather than treating it as raw text. + */ +static void test_broadcaster_sendBeacon_usesBeaconPortnum(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + strncpy(moduleConfig.mesh_beacon.broadcast_message, "portnum-check", sizeof(moduleConfig.mesh_beacon.broadcast_message) - 1); + moduleConfig.mesh_beacon.has_broadcast_offer_preset = true; + moduleConfig.mesh_beacon.broadcast_offer_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_SLOW; + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.sendBeacon(); + + TEST_ASSERT_EQUAL_UINT32(1, mockRouter->sentPackets.size()); + TEST_ASSERT_EQUAL(meshtastic_PortNum_MESH_BEACON_APP, mockRouter->sentPackets[0].decoded.portnum); +} + +/** + * Verify TEXT_MESSAGE_APP portnum is used when no offer content is present, even if + * broadcast_on_preset is set (that field governs which radio config to use for TX, not portnum). + * Important so standard clients display plain-text beacons without needing a MESH_BEACON_APP decoder. + */ +static void test_broadcaster_sendBeacon_fallsBackToTextMessagePortnum(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + const char *msg = "plain-text-beacon"; + strncpy(moduleConfig.mesh_beacon.broadcast_message, msg, sizeof(moduleConfig.mesh_beacon.broadcast_message) - 1); + // broadcast_on_preset set, but no offer — should still be TEXT_MESSAGE_APP + moduleConfig.mesh_beacon.has_broadcast_on_preset = true; + moduleConfig.mesh_beacon.broadcast_on_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_SLOW; + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.sendBeacon(); + + TEST_ASSERT_EQUAL_UINT32(1, mockRouter->sentPackets.size()); + const meshtastic_MeshPacket &p = mockRouter->sentPackets[0]; + TEST_ASSERT_EQUAL(meshtastic_PortNum_TEXT_MESSAGE_APP, p.decoded.portnum); + TEST_ASSERT_EQUAL_UINT32(strlen(msg), p.decoded.payload.size); + TEST_ASSERT_EQUAL_STRING_LEN(msg, (const char *)p.decoded.payload.bytes, p.decoded.payload.size); +} + +/** + * Verify the MESH_BEACON_APP payload decodes back to the original message string. + * Important to catch encode/decode regressions in the full sendBeacon → wire → pb_decode round-trip. + */ +static void test_broadcaster_sendBeacon_payloadDecodesCorrectly(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + const char *msg = "Greetings from the beacon"; + strncpy(moduleConfig.mesh_beacon.broadcast_message, msg, sizeof(moduleConfig.mesh_beacon.broadcast_message) - 1); + moduleConfig.mesh_beacon.has_broadcast_offer_preset = true; + moduleConfig.mesh_beacon.broadcast_offer_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_SLOW; + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.sendBeacon(); + + TEST_ASSERT_EQUAL_UINT32(1, mockRouter->sentPackets.size()); + const meshtastic_MeshPacket &p = mockRouter->sentPackets[0]; + meshtastic_MeshBeacon decoded = meshtastic_MeshBeacon_init_zero; + pb_istream_t stream = pb_istream_from_buffer(p.decoded.payload.bytes, p.decoded.payload.size); + bool ok = pb_decode(&stream, &meshtastic_MeshBeacon_msg, &decoded); + + TEST_ASSERT_TRUE_MESSAGE(ok, "Sent payload must decode without error"); + TEST_ASSERT_EQUAL_STRING(msg, decoded.message); +} + +/** + * Verify a beacon with offer fields but no message text is still emitted on MESH_BEACON_APP. + * Important because offer-only beacons are a valid use case that the early-return guard must not + * suppress. + */ +static void test_broadcaster_sendBeacon_offerOnly_isSent(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.has_broadcast_offer_preset = true; + moduleConfig.mesh_beacon.broadcast_offer_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_FAST; + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.sendBeacon(); + + TEST_ASSERT_EQUAL_UINT32(1, mockRouter->sentPackets.size()); + TEST_ASSERT_EQUAL(meshtastic_PortNum_MESH_BEACON_APP, mockRouter->sentPackets[0].decoded.portnum); +} + +/** + * Verify runOnce sends exactly one packet when broadcast_enabled is true. + * Important to confirm the OSThread timer callback drives the full send path end-to-end. + */ +static void test_broadcaster_runOnce_sendsWhenEnabled(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.flags |= MESH_BEACON_FLAG_BROADCAST_ENABLED; + moduleConfig.mesh_beacon.broadcast_interval_secs = 3600; + strncpy(moduleConfig.mesh_beacon.broadcast_message, "runOnce-enabled", + sizeof(moduleConfig.mesh_beacon.broadcast_message) - 1); + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.runOnce(); + + TEST_ASSERT_EQUAL_UINT32(1, mockRouter->sentPackets.size()); +} + +/** + * Verify runOnce transmits nothing when broadcast_enabled is false. + * Important to confirm the feature can be cleanly disabled via remote admin without rebooting. + */ +static void test_broadcaster_runOnce_silentWhenDisabled(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.flags &= ~MESH_BEACON_FLAG_BROADCAST_ENABLED; + strncpy(moduleConfig.mesh_beacon.broadcast_message, "runOnce-disabled", + sizeof(moduleConfig.mesh_beacon.broadcast_message) - 1); + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.runOnce(); + + TEST_ASSERT_EQUAL_UINT32(0, mockRouter->sentPackets.size()); +} + +// =========================================================================== +// Group 4: Listener — offer caching and guards +// =========================================================================== + +// Helper: build a decoded MESH_BEACON_APP packet carrying the given MeshBeacon. +static meshtastic_MeshPacket makeBeaconPacket(const meshtastic_MeshBeacon &b, NodeNum from = kRemoteNode) +{ + meshtastic_MeshPacket p = meshtastic_MeshPacket_init_zero; + p.from = from; + p.to = NODENUM_BROADCAST; + p.id = 0xDEAD0001; + p.which_payload_variant = meshtastic_MeshPacket_decoded_tag; + p.decoded.portnum = meshtastic_PortNum_MESH_BEACON_APP; + p.decoded.payload.size = + (pb_size_t)pb_encode_to_bytes(p.decoded.payload.bytes, sizeof(p.decoded.payload.bytes), &meshtastic_MeshBeacon_msg, &b); + return p; +} + +/** + * Verify a beacon carrying preset and region offer fields is stored in lastReceivedOffer. + * Important to confirm the client app's offer cache is populated correctly for join-offer UI flows. + */ +static void test_listener_receiveWithOffer_cachesOffer(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.flags |= MESH_BEACON_FLAG_LISTEN_ENABLED; + + MeshBeaconListenerModuleTestShim listener; + MeshBeaconListenerModule::lastReceivedOffer = {}; + + meshtastic_MeshBeacon b = meshtastic_MeshBeacon_init_zero; + strncpy(b.message, "Join us on US/MEDIUM_FAST", sizeof(b.message) - 1); + b.has_offer_preset = true; + b.offer_preset = meshtastic_Config_LoRaConfig_ModemPreset_MEDIUM_FAST; + b.offer_region = meshtastic_Config_LoRaConfig_RegionCode_US; + + meshtastic_MeshPacket mp = makeBeaconPacket(b); + listener.handleReceivedProtobuf(mp, &b); + + TEST_ASSERT_TRUE_MESSAGE(MeshBeaconListenerModule::lastReceivedOffer.valid, "Offer with preset must be cached"); + TEST_ASSERT_EQUAL(kRemoteNode, MeshBeaconListenerModule::lastReceivedOffer.sender); + TEST_ASSERT_EQUAL(meshtastic_Config_LoRaConfig_ModemPreset_MEDIUM_FAST, MeshBeaconListenerModule::lastReceivedOffer.preset); + TEST_ASSERT_EQUAL(meshtastic_Config_LoRaConfig_RegionCode_US, MeshBeaconListenerModule::lastReceivedOffer.region); +} + +/** + * Verify a beacon with a full ChannelSettings offer sets has_channel and copies the channel struct. + * Important because the client app checks has_channel before rendering a channel join offer. + */ +static void test_listener_receiveWithChannelOffer_setsHasChannel(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.flags |= MESH_BEACON_FLAG_LISTEN_ENABLED; + + MeshBeaconListenerModuleTestShim listener; + MeshBeaconListenerModule::lastReceivedOffer = {}; + + meshtastic_MeshBeacon b = meshtastic_MeshBeacon_init_zero; + strncpy(b.message, "Channel offer test", sizeof(b.message) - 1); + b.has_offer_channel = true; + b.offer_channel.channel_num = 5; + strncpy(b.offer_channel.name, "TestNet", sizeof(b.offer_channel.name) - 1); + + meshtastic_MeshPacket mp = makeBeaconPacket(b); + listener.handleReceivedProtobuf(mp, &b); + + TEST_ASSERT_TRUE(MeshBeaconListenerModule::lastReceivedOffer.valid); + TEST_ASSERT_TRUE_MESSAGE(MeshBeaconListenerModule::lastReceivedOffer.has_channel, + "has_channel must be set when offer_channel is present"); + TEST_ASSERT_EQUAL_UINT32(5, MeshBeaconListenerModule::lastReceivedOffer.channel.channel_num); +} + +/** + * Verify a beacon with neither message text nor offer fields is silently discarded. + * Important to avoid spurious cache updates and wasted inbox copies from empty-payload packets. + */ +static void test_listener_emptyMessageWithoutOffer_isDropped(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.flags |= MESH_BEACON_FLAG_LISTEN_ENABLED; + + MeshBeaconListenerModuleTestShim listener; + MeshBeaconListenerModule::lastReceivedOffer = {}; + + meshtastic_MeshBeacon b = meshtastic_MeshBeacon_init_zero; + // message field intentionally left blank + + meshtastic_MeshPacket mp = makeBeaconPacket(b); + listener.handleReceivedProtobuf(mp, &b); + + TEST_ASSERT_FALSE_MESSAGE(MeshBeaconListenerModule::lastReceivedOffer.valid, "Empty message must not update offer cache"); +} + +/** + * Verify a LONG_FAST offer (preset enum value 0) with no message still populates the offer cache. + * Important to guard the has_offer_preset fix — LONG_FAST must not be treated as 'no offer present'. + */ +static void test_listener_offerOnly_isCached(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.flags |= MESH_BEACON_FLAG_LISTEN_ENABLED; + + MeshBeaconListenerModuleTestShim listener; + MeshBeaconListenerModule::lastReceivedOffer = {}; + + meshtastic_MeshBeacon b = meshtastic_MeshBeacon_init_zero; + b.has_offer_preset = true; + b.offer_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_FAST; + + meshtastic_MeshPacket mp = makeBeaconPacket(b); + listener.handleReceivedProtobuf(mp, &b); + + TEST_ASSERT_TRUE(MeshBeaconListenerModule::lastReceivedOffer.valid); + TEST_ASSERT_EQUAL(meshtastic_Config_LoRaConfig_ModemPreset_LONG_FAST, MeshBeaconListenerModule::lastReceivedOffer.preset); +} + +/** + * Verify a null MeshBeacon pointer is handled gracefully and returns false without a crash. + * Important to guard against the ProtobufModule base class passing nullptr on a decode failure. + */ +static void test_listener_nullBeacon_isDropped(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.flags |= MESH_BEACON_FLAG_LISTEN_ENABLED; + + MeshBeaconListenerModuleTestShim listener; + MeshBeaconListenerModule::lastReceivedOffer = {}; + + meshtastic_MeshPacket mp = meshtastic_MeshPacket_init_zero; + bool result = listener.handleReceivedProtobuf(mp, nullptr); + + TEST_ASSERT_FALSE_MESSAGE(result, "Null beacon must return false"); + TEST_ASSERT_FALSE(MeshBeaconListenerModule::lastReceivedOffer.valid); +} + +/** + * Verify a text-only beacon (no offer fields set) does not mark the offer cache valid. + * Important to prevent the client from showing a join dialog in response to plain-text beacons. + */ +static void test_listener_receiveWithNoOffer_cacheStaysInvalid(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.flags |= MESH_BEACON_FLAG_LISTEN_ENABLED; + + MeshBeaconListenerModuleTestShim listener; + MeshBeaconListenerModule::lastReceivedOffer = {}; + + meshtastic_MeshBeacon b = meshtastic_MeshBeacon_init_zero; + strncpy(b.message, "No offer here", sizeof(b.message) - 1); + // has_offer_preset == false, has_offer_channel == false + + meshtastic_MeshPacket mp = makeBeaconPacket(b); + listener.handleReceivedProtobuf(mp, &b); + + TEST_ASSERT_FALSE_MESSAGE(MeshBeaconListenerModule::lastReceivedOffer.valid, "No offer fields → cache must stay invalid"); +} + +/** + * Verify the listener does NOT unwrap a combined beacon's text into a synthesized TEXT_MESSAGE_APP. + * The original MESH_BEACON_APP packet already reaches the client (the handler returns CONTINUE), so + * a beacon-aware client reads `message` directly from it — injecting a copy would only duplicate it, + * and re-injecting onto the mesh would amplify/re-attribute. So: nothing onto the mesh, nothing + * synthesized to the phone, and the handler must not consume the packet. + */ +static void test_listener_textMessage_notUnwrapped(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.flags |= MESH_BEACON_FLAG_LISTEN_ENABLED; + + MeshBeaconListenerModuleTestShim listener; + MeshBeaconListenerModule::lastReceivedOffer = {}; + + meshtastic_MeshBeacon b = meshtastic_MeshBeacon_init_zero; + strncpy(b.message, "hello mesh", sizeof(b.message) - 1); + + meshtastic_MeshPacket mp = makeBeaconPacket(b); + bool consumed = listener.handleReceivedProtobuf(mp, &b); + + // CONTINUE (not STOP): the original MESH_BEACON_APP keeps flowing to the client, which reads + // `message` from it — the simple path for a beacon-aware client. + TEST_ASSERT_FALSE_MESSAGE(consumed, "Listener must not consume the beacon; the original must reach the client"); + // Nothing re-injected onto the mesh. + TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, mockRouter->sentPackets.size(), + "Received beacon text must not be re-injected into the mesh"); + // No synthesized TEXT_MESSAGE_APP delivered to the phone (no duplicate of the beacon's text). + meshtastic_MeshPacket *toPhone = service->getForPhone(); + TEST_ASSERT_NULL_MESSAGE(toPhone, "Listener must not inject a duplicate text packet to the phone"); +} + +/** + * Verify wantPacket returns false for MESH_BEACON_APP when listen_enabled is false. + * Important to confirm the module opts out of processing when its config flag is cleared. + */ +static void test_listener_wantPacket_falseWhenDisabled(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.flags &= ~MESH_BEACON_FLAG_LISTEN_ENABLED; + + MeshBeaconListenerModuleTestShim listener; + + meshtastic_MeshPacket mp = meshtastic_MeshPacket_init_zero; + mp.decoded.portnum = meshtastic_PortNum_MESH_BEACON_APP; + + TEST_ASSERT_FALSE(listener.wantPacket(&mp)); +} + +/** + * Verify wantPacket returns true for MESH_BEACON_APP packets when listen_enabled is true. + * Important as a basic routing sanity check confirming the module is registered for its portnum. + */ +static void test_listener_wantPacket_trueWhenEnabled(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.flags |= MESH_BEACON_FLAG_LISTEN_ENABLED; + + MeshBeaconListenerModuleTestShim listener; + + meshtastic_MeshPacket mp = meshtastic_MeshPacket_init_zero; + mp.decoded.portnum = meshtastic_PortNum_MESH_BEACON_APP; + + TEST_ASSERT_TRUE(listener.wantPacket(&mp)); +} + +// =========================================================================== +// Group 6: Legacy split messages +// =========================================================================== + +/** + * Verify broadcast_legacy_split causes sendBeacon to emit exactly two packets when both + * text and offer content are present. + * Important to confirm the split path is wired end-to-end rather than short-circuiting. + */ +static void test_broadcaster_legacySplit_sendsTwoPackets(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.flags |= MESH_BEACON_FLAG_LEGACY_SPLIT; + strncpy(moduleConfig.mesh_beacon.broadcast_message, "split-text", sizeof(moduleConfig.mesh_beacon.broadcast_message) - 1); + moduleConfig.mesh_beacon.has_broadcast_offer_preset = true; + moduleConfig.mesh_beacon.broadcast_offer_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_SLOW; + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.sendBeacon(); + + TEST_ASSERT_EQUAL_UINT32_MESSAGE(2, mockRouter->sentPackets.size(), "Legacy split must emit exactly 2 packets"); +} + +/** + * Verify the first packet in a legacy-split send is MESH_BEACON_APP (the offer packet). + * Important so receivers without a MESH_BEACON_APP decoder still get the text from packet B. + */ +static void test_broadcaster_legacySplit_firstPacketIsBeaconApp(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.flags |= MESH_BEACON_FLAG_LEGACY_SPLIT; + strncpy(moduleConfig.mesh_beacon.broadcast_message, "split-offer-only", + sizeof(moduleConfig.mesh_beacon.broadcast_message) - 1); + moduleConfig.mesh_beacon.has_broadcast_offer_preset = true; + moduleConfig.mesh_beacon.broadcast_offer_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_SLOW; + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.sendBeacon(); + + TEST_ASSERT_EQUAL_UINT32(2, mockRouter->sentPackets.size()); + TEST_ASSERT_EQUAL(meshtastic_PortNum_MESH_BEACON_APP, mockRouter->sentPackets[0].decoded.portnum); +} + +/** + * Verify the MESH_BEACON_APP packet in a legacy-split send carries no message text. + * Important so peers' MESH_BEACON_APP handlers do not duplicate the text already in packet B. + */ +static void test_broadcaster_legacySplit_firstPacketHasNoMessageText(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.flags |= MESH_BEACON_FLAG_LEGACY_SPLIT; + strncpy(moduleConfig.mesh_beacon.broadcast_message, "hidden-in-split", + sizeof(moduleConfig.mesh_beacon.broadcast_message) - 1); + moduleConfig.mesh_beacon.has_broadcast_offer_preset = true; + moduleConfig.mesh_beacon.broadcast_offer_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_SLOW; + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.sendBeacon(); + + TEST_ASSERT_EQUAL_UINT32(2, mockRouter->sentPackets.size()); + const meshtastic_MeshPacket &pA = mockRouter->sentPackets[0]; + meshtastic_MeshBeacon decodedA = meshtastic_MeshBeacon_init_zero; + pb_istream_t stream = pb_istream_from_buffer(pA.decoded.payload.bytes, pA.decoded.payload.size); + pb_decode(&stream, &meshtastic_MeshBeacon_msg, &decodedA); + TEST_ASSERT_EQUAL_STRING_MESSAGE("", decodedA.message, "Offer-only MESH_BEACON_APP must have empty message"); +} + +/** + * Verify the second packet in a legacy-split send is TEXT_MESSAGE_APP containing the message text. + * Important so legacy clients that only handle TEXT_MESSAGE_APP receive the human-readable text. + */ +static void test_broadcaster_legacySplit_secondPacketIsTextMessage(void) +{ + resetConfig(); + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.flags |= MESH_BEACON_FLAG_LEGACY_SPLIT; + const char *msg = "split-B-text"; + strncpy(moduleConfig.mesh_beacon.broadcast_message, msg, sizeof(moduleConfig.mesh_beacon.broadcast_message) - 1); + moduleConfig.mesh_beacon.has_broadcast_offer_preset = true; + moduleConfig.mesh_beacon.broadcast_offer_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_SLOW; + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.sendBeacon(); + + TEST_ASSERT_EQUAL_UINT32(2, mockRouter->sentPackets.size()); + const meshtastic_MeshPacket &pB = mockRouter->sentPackets[1]; + TEST_ASSERT_EQUAL(meshtastic_PortNum_TEXT_MESSAGE_APP, pB.decoded.portnum); + TEST_ASSERT_EQUAL_STRING_LEN(msg, (const char *)pB.decoded.payload.bytes, pB.decoded.payload.size); +} + +// =========================================================================== +// Group 7: Beacon-channel PSK swap (broadcast_on_channel override) +// =========================================================================== + +/** + * When broadcast_on_channel overrides the primary channel's name/PSK, the packet must be encrypted + * on the BEACON channel, not the primary. perhapsEncode keys off the primary slot, so sendBeaconPacket + * temporarily installs the beacon channel there for the send and restores it after. Verify both: the + * primary slot IS the beacon channel during send(), and it is restored afterwards (no leak). + */ +static void test_broadcaster_channelPskOverride_swapsBeaconChannelAndRestores(void) +{ + resetConfig(); + static const uint8_t homePsk[16] = {0xAA, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}; + installTestPrimaryChannel("Home", homePsk, sizeof(homePsk)); + + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.has_broadcast_offer_preset = true; // gives the beacon radio content to send + moduleConfig.mesh_beacon.broadcast_offer_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_SLOW; + moduleConfig.mesh_beacon.has_broadcast_on_channel = true; + strncpy(moduleConfig.mesh_beacon.broadcast_on_channel.name, "BeaconCh", + sizeof(moduleConfig.mesh_beacon.broadcast_on_channel.name) - 1); + static const uint8_t beaconPsk[16] = {0xBB, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f}; + moduleConfig.mesh_beacon.broadcast_on_channel.psk.size = sizeof(beaconPsk); + memcpy(moduleConfig.mesh_beacon.broadcast_on_channel.psk.bytes, beaconPsk, sizeof(beaconPsk)); + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.sendBeacon(); + + // During send(), the primary slot must hold the BEACON channel (so encryption uses its PSK). + TEST_ASSERT_TRUE_MESSAGE(mockRouter->primaryAtSend.size() >= 1, "expected at least one send"); + const meshtastic_ChannelSettings &atSend = mockRouter->primaryAtSend[0]; + TEST_ASSERT_EQUAL_STRING_MESSAGE("BeaconCh", atSend.name, "primary must be the beacon channel during send"); + TEST_ASSERT_EQUAL_UINT(sizeof(beaconPsk), atSend.psk.size); + TEST_ASSERT_EQUAL_UINT8_MESSAGE(0xBB, atSend.psk.bytes[0], "encryption must use the beacon channel PSK"); + + // After send(), the primary channel must be restored to the original (no leak into normal traffic). + const meshtastic_ChannelSettings &after = channels.getByIndex(channels.getPrimaryIndex()).settings; + TEST_ASSERT_EQUAL_STRING_MESSAGE("Home", after.name, "primary channel must be restored after send"); + TEST_ASSERT_EQUAL_UINT(sizeof(homePsk), after.psk.size); + TEST_ASSERT_EQUAL_UINT8(0xAA, after.psk.bytes[0]); +} + +/** + * Without a broadcast_on_channel override, the beacon must transmit on the primary channel unchanged + * (no swap). Guards against the swap firing — and churning the channel table — when it isn't needed. + */ +static void test_broadcaster_noChannelOverride_doesNotSwapPrimary(void) +{ + resetConfig(); + static const uint8_t homePsk[16] = {0xAA, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}; + installTestPrimaryChannel("Home", homePsk, sizeof(homePsk)); + + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.has_broadcast_offer_preset = true; + moduleConfig.mesh_beacon.broadcast_offer_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_SLOW; + // No broadcast_on_channel override. + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.sendBeacon(); + + TEST_ASSERT_TRUE_MESSAGE(mockRouter->primaryAtSend.size() >= 1, "expected at least one send"); + TEST_ASSERT_EQUAL_STRING_MESSAGE("Home", mockRouter->primaryAtSend[0].name, + "primary must stay unchanged when no channel override is set"); +} + +/** + * A broadcast_target whose channel_index points at a configured table slot must transmit encrypted + * on THAT slot's channel (name + PSK), not the primary. Verifies the slot-index → channel-table + * resolution introduced when BroadcastTarget dropped its embedded ChannelSettings. + */ +static void test_broadcaster_targetChannelIndex_usesTableSlot(void) +{ + resetConfig(); + static const uint8_t homePsk[16] = {0xAA, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}; + static const uint8_t beaconPsk[16] = {0xBB, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f}; + installTestPrimaryChannel("Home", homePsk, sizeof(homePsk)); + installTestSecondaryChannel(1, "BeaconNet", beaconPsk, sizeof(beaconPsk)); + + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.has_broadcast_offer_preset = true; // content to send + moduleConfig.mesh_beacon.broadcast_offer_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_SLOW; + moduleConfig.mesh_beacon.broadcast_targets_count = 1; + moduleConfig.mesh_beacon.broadcast_targets[0].has_channel_index = true; + moduleConfig.mesh_beacon.broadcast_targets[0].channel_index = 1; + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.sendBeacon(); + + TEST_ASSERT_TRUE_MESSAGE(mockRouter->primaryAtSend.size() >= 1, "expected at least one send"); + TEST_ASSERT_EQUAL_STRING_MESSAGE("BeaconNet", mockRouter->primaryAtSend[0].name, + "beacon must be encrypted on the referenced slot's channel"); + // Primary slot restored to home after send (no leak). + TEST_ASSERT_EQUAL_STRING("Home", channels.getByIndex(channels.getPrimaryIndex()).settings.name); +} + +/** + * A broadcast_target whose channel_index points at a BLANK table slot (no name, no PSK) must fall + * back to the default channel for the preset rather than borrowing the primary's name/PSK with a + * clobbered channel_num. Guards the blank-slot handling for multi-target configs. + */ +static void test_broadcaster_targetChannelIndex_blankSlotFallsBackToPreset(void) +{ + resetConfig(); + static const uint8_t homePsk[16] = {0xAA, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}; + installTestPrimaryChannel("Home", homePsk, sizeof(homePsk)); + installTestSecondaryChannel(1, nullptr, nullptr, 0); // blank slot: no name, no PSK + + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.has_broadcast_offer_preset = true; + moduleConfig.mesh_beacon.broadcast_offer_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_SLOW; + moduleConfig.mesh_beacon.broadcast_targets_count = 1; + moduleConfig.mesh_beacon.broadcast_targets[0].has_channel_index = true; + moduleConfig.mesh_beacon.broadcast_targets[0].channel_index = 1; + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.sendBeacon(); + + // Exactly one beacon goes out, and a blank slot does NOT trigger a crypto swap — the primary + // stays "Home" (no garbage / no clobber), matching the no-channel-override default-for-preset path. + TEST_ASSERT_EQUAL_UINT32(1, mockRouter->sentPackets.size()); + TEST_ASSERT_TRUE_MESSAGE(mockRouter->primaryAtSend.size() >= 1, "expected at least one send"); + TEST_ASSERT_EQUAL_STRING_MESSAGE("Home", mockRouter->primaryAtSend[0].name, + "blank slot must not swap the beacon onto a borrowed channel"); +} + +/** + * Two broadcast_targets that resolve to the same effective radio config (same preset/region/channel) + * must produce only ONE beacon — the payload is identical, so re-broadcasting wastes airtime. + */ +static void test_broadcaster_duplicateTargets_dedupedToOnePacket(void) +{ + resetConfig(); + static const uint8_t homePsk[16] = {0xAA, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}; + installTestPrimaryChannel("Home", homePsk, sizeof(homePsk)); + + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.has_broadcast_offer_preset = true; + moduleConfig.mesh_beacon.broadcast_offer_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_SLOW; + moduleConfig.mesh_beacon.broadcast_targets_count = 2; + for (int i = 0; i < 2; i++) { + moduleConfig.mesh_beacon.broadcast_targets[i].has_preset = true; + moduleConfig.mesh_beacon.broadcast_targets[i].preset = meshtastic_Config_LoRaConfig_ModemPreset_NARROW_SLOW; + moduleConfig.mesh_beacon.broadcast_targets[i].region = meshtastic_Config_LoRaConfig_RegionCode_UNSET; + } + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.sendBeacon(); + + TEST_ASSERT_EQUAL_UINT32_MESSAGE(1, mockRouter->sentPackets.size(), "duplicate targets must collapse to one beacon"); +} + +/** + * Two distinct broadcast_targets (different presets) must BOTH be sent — dedup must not over-collapse. + */ +static void test_broadcaster_distinctTargets_bothSent(void) +{ + resetConfig(); + static const uint8_t homePsk[16] = {0xAA, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}; + installTestPrimaryChannel("Home", homePsk, sizeof(homePsk)); + + moduleConfig.has_mesh_beacon = true; + moduleConfig.mesh_beacon.has_broadcast_offer_preset = true; + moduleConfig.mesh_beacon.broadcast_offer_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_SLOW; + moduleConfig.mesh_beacon.broadcast_targets_count = 2; + moduleConfig.mesh_beacon.broadcast_targets[0].has_preset = true; + moduleConfig.mesh_beacon.broadcast_targets[0].preset = meshtastic_Config_LoRaConfig_ModemPreset_NARROW_SLOW; + moduleConfig.mesh_beacon.broadcast_targets[1].has_preset = true; + moduleConfig.mesh_beacon.broadcast_targets[1].preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_FAST; + + MeshBeaconBroadcastModuleTestShim bcast; + bcast.sendBeacon(); + + TEST_ASSERT_EQUAL_UINT32_MESSAGE(2, mockRouter->sentPackets.size(), "distinct targets must each be sent"); +} + +} // namespace + +// =========================================================================== +// Unity lifecycle +// =========================================================================== + +void setUp(void) +{ + testAirTime = new AirTime(); + airTime = testAirTime; + + mockSvc = new MockMeshService(); + service = mockSvc; + + mockRouter = new MockRouter(); + router = mockRouter; + + testAdmin = new AdminModuleTestShim(); +} + +void tearDown(void) +{ + meshBeaconBroadcastModule = nullptr; + + delete testAdmin; + testAdmin = nullptr; + + // Drain any packets the listener delivered via sendToPhone() (toPhoneQueue takes ownership and + // nothing else dequeues them in tests) so they are returned to packetPool — otherwise they leak + // and LeakSanitizer aborts the process at exit. + if (mockSvc) { + meshtastic_MeshPacket *p; + while ((p = mockSvc->getForPhone()) != nullptr) + mockSvc->releaseToPool(p); + } + + service = nullptr; + delete mockSvc; + mockSvc = nullptr; + + router = nullptr; + delete mockRouter; + mockRouter = nullptr; + + airTime = nullptr; + delete testAirTime; + testAirTime = nullptr; +} + +BEACON_TEST_ENTRY void setup() +{ + delay(10); + initializeTestEnvironment(); + UNITY_BEGIN(); + + printf("\n=== AdminModule config validation ===\n"); + + RUN_TEST(test_adminValidation_turboPresetOnEU868_isCleared); + RUN_TEST(test_adminValidation_longTurboPresetOnEU868_isCleared); + RUN_TEST(test_adminValidation_turboPresetOnUS_isAccepted); + RUN_TEST(test_adminValidation_unknownOfferRegion_isCleared); + RUN_TEST(test_adminValidation_validOfferRegion_isPreserved); + RUN_TEST(test_adminValidation_targetUnknownRegion_isCleared); + RUN_TEST(test_adminValidation_targetInvalidPresetForRegion_isCleared); + RUN_TEST(test_adminValidation_targetValidPresetForRegion_isPreserved); + RUN_TEST(test_adminValidation_targetChannelIndexOutOfRange_isCleared); + RUN_TEST(test_adminValidation_targetChannelIndexInRange_isPreserved); + RUN_TEST(test_adminValidation_messageTooLong_isTruncatedAt100); + RUN_TEST(test_adminValidation_intervalTooLow_isClamped); + RUN_TEST(test_adminValidation_intervalTooHigh_isPreserved); + RUN_TEST(test_adminValidation_intervalZero_isNotClamped); + RUN_TEST(test_adminValidation_longFastOfferPreset_isPreserved); + RUN_TEST(test_adminValidation_validSave_invalidatesCache); + + printf("\n=== Broadcaster payload cache ===\n"); + + RUN_TEST(test_broadcaster_rebuildCache_producesNonEmptyPayload); + RUN_TEST(test_broadcaster_rebuildCache_payloadDecodesCorrectly); + RUN_TEST(test_broadcaster_rebuildCache_offerFieldsEncoded); + RUN_TEST(test_broadcaster_invalidateCache_setsDirtyFlag); + RUN_TEST(test_broadcaster_rebuildCache_idempotent); + + printf("\n=== Broadcaster sendBeacon ===\n"); + + RUN_TEST(test_broadcaster_sendBeacon_fromIsLocalNodeWhenUnset); + RUN_TEST(test_broadcaster_sendBeacon_fromIsCustomNodeWhenSet); + RUN_TEST(test_broadcaster_sendBeacon_addressedToBroadcast); + RUN_TEST(test_broadcaster_sendBeacon_usesBeaconPortnum); + RUN_TEST(test_broadcaster_sendBeacon_fallsBackToTextMessagePortnum); + RUN_TEST(test_broadcaster_sendBeacon_payloadDecodesCorrectly); + RUN_TEST(test_broadcaster_sendBeacon_offerOnly_isSent); + RUN_TEST(test_broadcaster_runOnce_sendsWhenEnabled); + RUN_TEST(test_broadcaster_runOnce_silentWhenDisabled); + + printf("\n=== Listener offer caching ===\n"); + + RUN_TEST(test_listener_receiveWithOffer_cachesOffer); + RUN_TEST(test_listener_receiveWithChannelOffer_setsHasChannel); + RUN_TEST(test_listener_emptyMessageWithoutOffer_isDropped); + RUN_TEST(test_listener_offerOnly_isCached); + RUN_TEST(test_listener_nullBeacon_isDropped); + RUN_TEST(test_listener_receiveWithNoOffer_cacheStaysInvalid); + RUN_TEST(test_listener_textMessage_notUnwrapped); + RUN_TEST(test_listener_wantPacket_falseWhenDisabled); + RUN_TEST(test_listener_wantPacket_trueWhenEnabled); + + printf("\n=== Legacy split messages ===\n"); + + RUN_TEST(test_broadcaster_legacySplit_sendsTwoPackets); + RUN_TEST(test_broadcaster_legacySplit_firstPacketIsBeaconApp); + RUN_TEST(test_broadcaster_legacySplit_firstPacketHasNoMessageText); + RUN_TEST(test_broadcaster_legacySplit_secondPacketIsTextMessage); + + printf("\n=== Beacon-channel PSK swap ===\n"); + + RUN_TEST(test_broadcaster_channelPskOverride_swapsBeaconChannelAndRestores); + RUN_TEST(test_broadcaster_noChannelOverride_doesNotSwapPrimary); + RUN_TEST(test_broadcaster_targetChannelIndex_usesTableSlot); + RUN_TEST(test_broadcaster_targetChannelIndex_blankSlotFallsBackToPreset); + RUN_TEST(test_broadcaster_duplicateTargets_dedupedToOnePacket); + RUN_TEST(test_broadcaster_distinctTargets_bothSent); + + exit(UNITY_END()); +} + +BEACON_TEST_ENTRY void loop() {} + +#else // MESHTASTIC_EXCLUDE_BEACON + +void setUp(void) {} +void tearDown(void) {} + +BEACON_TEST_ENTRY void setup() +{ + initializeTestEnvironment(); + UNITY_BEGIN(); + exit(UNITY_END()); +} + +BEACON_TEST_ENTRY void loop() {} + +#endif diff --git a/userPrefs.jsonc b/userPrefs.jsonc index 118588e19..4df50abca 100644 --- a/userPrefs.jsonc +++ b/userPrefs.jsonc @@ -58,6 +58,37 @@ // "USERPREFS_MQTT_ENCRYPTION_ENABLED": "true", // "USERPREFS_MQTT_TLS_ENABLED": "false", // "USERPREFS_MQTT_ROOT_TOPIC": "event/REPLACEME", + // "USERPREFS_MESH_BEACON_LISTEN_ENABLED": "true", // Accept incoming beacons (default: on) + // "USERPREFS_MESH_BEACON_BROADCAST_ENABLED": "true", // Periodically broadcast beacons from this node + // "USERPREFS_MESH_BEACON_MESSAGE": "Join us on NarrowSlow!!!", // Text payload included in every beacon broadcast + // "USERPREFS_MESH_BEACON_INTERVAL_SECS": "3600", // Broadcast interval in seconds (min 3600 = 1 hour) + // "USERPREFS_MESH_BEACON_OFFER_PRESET": "meshtastic_Config_LoRaConfig_ModemPreset_NARROW_SLOW", // Modem preset advertised in the beacon payload (listeners can adopt this) + // "USERPREFS_MESH_BEACON_OFFER_REGION": "meshtastic_Config_LoRaConfig_RegionCode_EU_N_868", // Region advertised in the beacon payload + // "USERPREFS_MESH_BEACON_OFFER_CHANNEL_NAME": "'MyChannel'", // Channel name advertised in the beacon payload + // "USERPREFS_MESH_BEACON_OFFER_CHANNEL_PSK": "{ 0x38, 0x4b, 0xbc, 0xc0, 0x1d, 0xc0, 0x22, 0xd1, 0x81, 0xbf, 0x36, 0xb8, 0x61, 0x21, 0xe1, 0xfb, 0x96, 0xb7, 0x2e, 0x55, 0xbf, 0x74, 0x22, 0x7e, 0x9d, 0x6a, 0xfb, 0x48, 0xd6, 0x4c, 0xb1, 0xa1 }", // PSK for the offered channel (32-byte AES-256) + // "USERPREFS_MESH_BEACON_ON_PRESET": "meshtastic_Config_LoRaConfig_ModemPreset_LONG_FAST", // Modem preset to use when transmitting beacons (radio temporarily switched for TX) + // "USERPREFS_MESH_BEACON_ON_REGION": "meshtastic_Config_LoRaConfig_RegionCode_EU_868", // Region to use when transmitting beacons + // "USERPREFS_MESH_BEACON_ON_CHANNEL_NAME": "'LongFast'", // Channel name to use on the TX radio config - uses default if unset. + // "USERPREFS_MESH_BEACON_ON_CHANNEL_PSK": "{ 0x01 }", // PSK for the TX channel (0x01 = Meshtastic default PSK) + // "USERPREFS_MESH_BEACON_ON_CHANNEL_NUM": "0", // LoRa channel/frequency slot to use on the TX radio config - zero is default, 20 is standard for US LongFast, etc. + // "USERPREFS_MESH_BEACON_LEGACY_SPLIT": "true", // When both text and offer are present, split into a separate MESH_BEACON_APP (offer) and TEXT_MESSAGE_APP (text) for legacy client compatibility + // Multi-target broadcast: when any TARGET__* key is set, broadcast_targets overrides the + // single-target broadcast_on_* fields above. Each target transmits its own copy of the beacon. + // Up to 4 targets (0-3). Only TARGET_0 is used here; uncomment TARGET_1 to add a second preset. + // CHANNEL_INDEX references a slot in the device's channel table (0..MAX_NUM_CHANNELS-1); that + // channel must already be configured on the node (its key is needed to encrypt the beacon). + // "USERPREFS_MESH_BEACON_TARGET_0_PRESET": "meshtastic_Config_LoRaConfig_ModemPreset_LONG_FAST", + // "USERPREFS_MESH_BEACON_TARGET_0_REGION": "meshtastic_Config_LoRaConfig_RegionCode_EU_868", + // "USERPREFS_MESH_BEACON_TARGET_0_CHANNEL_INDEX": "0", // device channel-table slot to transmit on + // "USERPREFS_MESH_BEACON_TARGET_1_PRESET": "meshtastic_Config_LoRaConfig_ModemPreset_NARROW_SLOW", + // "USERPREFS_MESH_BEACON_TARGET_1_REGION": "meshtastic_Config_LoRaConfig_RegionCode_EU_N_868", + // "USERPREFS_MESH_BEACON_TARGET_1_CHANNEL_INDEX": "1", + // "USERPREFS_MESH_BEACON_TARGET_2_PRESET": "meshtastic_Config_LoRaConfig_ModemPreset_MEDIUM_FAST", + // "USERPREFS_MESH_BEACON_TARGET_2_REGION": "meshtastic_Config_LoRaConfig_RegionCode_EU_868", + // "USERPREFS_MESH_BEACON_TARGET_2_CHANNEL_INDEX": "2", + // "USERPREFS_MESH_BEACON_TARGET_3_PRESET": "meshtastic_Config_LoRaConfig_ModemPreset_SHORT_FAST", + // "USERPREFS_MESH_BEACON_TARGET_3_REGION": "meshtastic_Config_LoRaConfig_RegionCode_EU_868", + // "USERPREFS_MESH_BEACON_TARGET_3_CHANNEL_INDEX": "3", // "USERPREFS_RINGTONE_NAG_SECS": "60", // "USERPREFS_NODEINFO_REPLY_SUPPRESS_SECS": "43200", // "USERPREFS_UI_TEST_LOG": "true", // Test-only: emits `Screen: frame N/M name=... reason=...` log per UI transition (for the mcp-server ui test tier); off in release builds. From 25c71cb9f13caf339eaeaea85df90005a545dc12 Mon Sep 17 00:00:00 2001 From: HarukiToreda <116696711+HarukiToreda@users.noreply.github.com> Date: Sat, 27 Jun 2026 15:35:03 -0400 Subject: [PATCH 44/86] InkHUD: add offline map tile backgrounds and zoom controls (#10785) * InkHUD: map tile background, zoom controls, and GPS live tracking - Map background tiles are now rendered on the display when available, compressed with LZ4 to keep flash usage low - If no map tiles are loaded, the map applets behave exactly as before - Zoom in, zoom out, and reset zoom (back to auto-fit) are accessible from the menu, and only appear when the menu is opened from a map screen - The map updates automatically whenever GPS gets a new position or your phone shares a location - The Positions and Favorites map applets now also refresh when mesh position packets arrive for your own node - The Favorites map now shows your position on the map even if you have no favorites yet * README Update * Update MapApplet.cpp * Zoom fixed * Zoom with no tiles fix * CI fix --- src/graphics/niche/InkHUD/Applet.h | 2 + .../InkHUD/Applets/Bases/Map/MapApplet.cpp | 499 +++++++++++++++--- .../InkHUD/Applets/Bases/Map/MapApplet.h | 39 +- .../niche/InkHUD/Applets/Bases/Map/MapTile.h | 9 + .../InkHUD/Applets/System/Menu/MenuAction.h | 4 + .../InkHUD/Applets/System/Menu/MenuApplet.cpp | 36 ++ .../User/FavoritesMap/FavoritesMapApplet.cpp | 10 + .../User/FavoritesMap/FavoritesMapApplet.h | 3 + .../Applets/User/Positions/PositionsApplet.h | 2 + src/graphics/niche/InkHUD/docs/README.md | 14 + 10 files changed, 533 insertions(+), 85 deletions(-) create mode 100644 src/graphics/niche/InkHUD/Applets/Bases/Map/MapTile.h diff --git a/src/graphics/niche/InkHUD/Applet.h b/src/graphics/niche/InkHUD/Applet.h index 6b727f273..2aa5bc640 100644 --- a/src/graphics/niche/InkHUD/Applet.h +++ b/src/graphics/niche/InkHUD/Applet.h @@ -128,6 +128,8 @@ class Applet : public GFX virtual bool approveNotification(Notification &n); // Allow an applet to veto a notification + virtual class MapApplet *asMapApplet() { return nullptr; } // Returns non-null only for MapApplet and its subclasses + static uint16_t getHeaderHeight(); // How tall the "standard" applet header is static AppletFont fontSmall, fontMedium, fontLarge; // The general purpose fonts, used by all applets diff --git a/src/graphics/niche/InkHUD/Applets/Bases/Map/MapApplet.cpp b/src/graphics/niche/InkHUD/Applets/Bases/Map/MapApplet.cpp index 881371e2d..35b0f567e 100644 --- a/src/graphics/niche/InkHUD/Applets/Bases/Map/MapApplet.cpp +++ b/src/graphics/niche/InkHUD/Applets/Bases/Map/MapApplet.cpp @@ -1,18 +1,367 @@ #ifdef MESHTASTIC_INCLUDE_INKHUD #include "./MapApplet.h" +#include "./MapTile.h" + +#include +#include using namespace NicheGraphics; +bool InkHUD::MapApplet::s_zoomLocked = false; +int InkHUD::MapApplet::s_lockedZoom = -1; +int InkHUD::MapApplet::s_lastRenderedZoom = -1; +int InkHUD::MapApplet::s_autoFitZoom = -1; + +// Observe GPS position updates so the map redraws whenever a new location arrives. +InkHUD::MapApplet::MapApplet() +{ + if (gpsStatus) + gpsStatusObserver.observe(&gpsStatus->onNewStatus); +} + +int InkHUD::MapApplet::onGpsStatusUpdate(const meshtastic::Status *status) +{ + if (status->getStatusType() != STATUS_TYPE_GPS) + return 0; + if (!isActive() || !gpsStatus->getHasLock()) + return 0; + + requestUpdate(); + return 0; +} + +// Zoom in one step from the current display zoom. +void InkHUD::MapApplet::zoomIn() +{ + int baseZoom = s_zoomLocked ? s_lockedZoom : s_lastRenderedZoom; + if (baseZoom < 0) + return; + + if (map_tile_count == 0) { + if (baseZoom < ZOOM_MAX_NO_TILES) { + s_lockedZoom = baseZoom + 1; + s_zoomLocked = true; + } + return; + } + + // Jump to the next tile zoom strictly above current, not just +1 + int next = -1; + for (int i = 0; i < map_tile_count; i++) { + int z = map_tile_zooms[i]; + if (z > baseZoom && (next < 0 || z < next)) + next = z; + } + if (next < 0) + return; + + s_lockedZoom = next; + s_zoomLocked = true; +} + +void InkHUD::MapApplet::resetZoom() +{ + s_zoomLocked = false; + s_lockedZoom = -1; +} + +bool InkHUD::MapApplet::canZoomIn() const +{ + if (s_lastRenderedZoom < 0) + return false; + int ref = s_zoomLocked ? s_lockedZoom : s_lastRenderedZoom; + if (map_tile_count == 0) + return ref < ZOOM_MAX_NO_TILES; + for (int i = 0; i < map_tile_count; i++) { + if (map_tile_zooms[i] > ref) + return true; + } + return false; +} + +void InkHUD::MapApplet::zoomOut() +{ + int baseZoom = s_zoomLocked ? s_lockedZoom : s_lastRenderedZoom; + if (baseZoom < 0) { + s_zoomLocked = false; + s_lockedZoom = -1; + return; + } + + if (map_tile_count == 0) { + int floor = (s_autoFitZoom >= 0) ? s_autoFitZoom : baseZoom; + if (baseZoom > floor) { + s_lockedZoom = baseZoom - 1; + s_zoomLocked = true; + } else { + s_zoomLocked = false; + s_lockedZoom = -1; + } + return; + } + + // Jump to the next tile zoom strictly below current, not just -1 + int next = -1; + for (int i = 0; i < map_tile_count; i++) { + int z = map_tile_zooms[i]; + if (z < baseZoom && (next < 0 || z > next)) + next = z; + } + if (next < 0) { + s_zoomLocked = false; + s_lockedZoom = -1; + return; + } + + s_lockedZoom = next; + s_zoomLocked = true; +} + +bool InkHUD::MapApplet::canZoomOut() const +{ + if (s_lastRenderedZoom < 0) + return false; + int ref = s_zoomLocked ? s_lockedZoom : s_lastRenderedZoom; + if (map_tile_count == 0) + return s_autoFitZoom >= 0 ? ref > s_autoFitZoom : false; + for (int i = 0; i < map_tile_count; i++) { + if (map_tile_zooms[i] < ref) + return true; + } + return false; +} + +// Raw LZ4 block decompressor. Returns bytes written, or -1 on error. +static int lz4_decompress(const uint8_t *src, int src_len, uint8_t *dst, int dst_cap) +{ + const uint8_t *s = src; + const uint8_t *s_end = src + src_len; + uint8_t *d = dst; + const uint8_t *d_end = dst + dst_cap; + while (s < s_end) { + uint8_t token = *s++; + int lit_len = (token >> 4) & 0xF; + if (lit_len == 15) { + uint8_t x; + do { + x = *s++; + lit_len += x; + } while (x == 255 && s < s_end); + } + if (d + lit_len > d_end || s + lit_len > s_end) + return -1; + memcpy(d, s, lit_len); + d += lit_len; + s += lit_len; + if (s >= s_end) + break; + if (s + 2 > s_end) + return -1; + int offset = (int)s[0] | ((int)s[1] << 8); + s += 2; + if (offset == 0 || d - offset < dst) + return -1; + int mat_len = (token & 0xF) + 4; + if (mat_len == 4 + 15) { + uint8_t x; + do { + x = *s++; + mat_len += x; + } while (x == 255 && s < s_end); + } + if (d + mat_len > d_end) + return -1; + const uint8_t *m = d - offset; + for (int i = 0; i < mat_len; i++) + *d++ = m[i]; + } + return (int)(d - dst); +} + +// Tiles are 1 bit/pixel, column-major: [bx=0..31][y=0..255], 8 pixels per byte. +static uint8_t s_tileCacheBuffer[8192]; + +static const uint8_t *decodeSparseTile(int tileIndex) +{ + int n = lz4_decompress(map_tile_data[tileIndex], map_tile_sizes[tileIndex], s_tileCacheBuffer, sizeof(s_tileCacheBuffer)); + return n == sizeof(s_tileCacheBuffer) ? s_tileCacheBuffer : nullptr; +} + +// Draw tiles centered on latCenter/lngCenter. Falls back to the nearest available zoom if +// no tiles exist at exactly zoom (upsamples), enabling smooth zoom steps. +void InkHUD::MapApplet::drawMapTileBackground(int zoom) +{ + if (map_tile_count == 0 || metersToPx <= 0.0f) + return; + + const float R = 6378137.0f; + const float latRad = latCenter * DEG_TO_RAD; + const float mpp = (2.0f * M_PI * R / (256.0f * (float)(1 << zoom))) * cosf(latRad); + const float worldPxPerScreenPx = 1.0f / (metersToPx * mpp); + + // Find best tile zoom: highest available <= zoom, or lowest available if none below. + int tileZoom = -1; + for (int i = 0; i < map_tile_count; i++) { + int z = map_tile_zooms[i]; + if (z <= zoom && (tileZoom < 0 || z > tileZoom)) + tileZoom = z; + } + if (tileZoom < 0) { + for (int i = 0; i < map_tile_count; i++) { + int z = map_tile_zooms[i]; + if (tileZoom < 0 || z < tileZoom) + tileZoom = z; + } + } + if (tileZoom < 0) + return; + + // Convert screen-pixel movement into tileZoom coordinate space. + // When tileZoom < zoom, tile pixels are upsampled (each tile pixel covers >1 screen px). + const float tileWorldPx = worldPxPerScreenPx * ((float)(1 << tileZoom) / (float)(1 << zoom)); + + const float sinLat = sinf(latRad); + const float gpxX = ((lngCenter + 180.0f) / 360.0f) * (float)(1 << tileZoom) * 256.0f; + const float gpxY = (0.5f - logf((1.0f + sinLat) / (1.0f - sinLat)) / (4.0f * M_PI)) * (float)(1 << tileZoom) * 256.0f; + + const float minWx = gpxX - width() * 0.5f * tileWorldPx; + const float maxWx = gpxX + width() * 0.5f * tileWorldPx; + const float minWy = gpxY - height() * 0.5f * tileWorldPx; + const float maxWy = gpxY + height() * 0.5f * tileWorldPx; + + for (int i = 0; i < map_tile_count; i++) { + if (map_tile_zooms[i] != tileZoom) + continue; + + const int tx = map_tile_tx[i]; + const int ty = map_tile_ty[i]; + const float tileMinWx = tx * 256.0f; + const float tileMaxWx = tileMinWx + 256.0f; + const float tileMinWy = ty * 256.0f; + const float tileMaxWy = tileMinWy + 256.0f; + if (tileMaxWx < minWx || tileMinWx > maxWx || tileMaxWy < minWy || tileMinWy > maxWy) + continue; + + const uint8_t *tile = decodeSparseTile(i); + if (!tile) + continue; + + const int sxStart = max(0, (int)floorf(((tileMinWx - gpxX) / tileWorldPx) + width() * 0.5f)); + const int sxEnd = min(width() - 1, (int)ceilf(((tileMaxWx - gpxX) / tileWorldPx) + width() * 0.5f) - 1); + const int syStart = max(0, (int)floorf(((tileMinWy - gpxY) / tileWorldPx) + height() * 0.5f)); + const int syEnd = min(height() - 1, (int)ceilf(((tileMaxWy - gpxY) / tileWorldPx) + height() * 0.5f) - 1); + + for (int sy = syStart; sy <= syEnd; sy++) { + const float wy = gpxY + (sy - height() * 0.5f) * tileWorldPx; + const int py = (int)(wy - tileMinWy); + if (py < 0 || py > 255) + continue; + + for (int sx = sxStart; sx <= sxEnd; sx++) { + const float wx = gpxX + (sx - width() * 0.5f) * tileWorldPx; + const int px = (int)(wx - tileMinWx); + if (px < 0 || px > 255) + continue; + + if (!(tile[(px / 8) * 256 + py] & (1 << (px % 8)))) + continue; + + drawPixel(sx, sy, BLACK); + } + } + } +} + void InkHUD::MapApplet::onRender(bool full) { - // Abort if no markers to render - if (!enoughMarkers()) { + // Map center is always the node centroid — tiles are background only. + getMapCenter(&latCenter, &lngCenter); + calculateAllMarkers(); + + // Show placeholder only if we have no position at all — no tiles, no own node + if (!enoughMarkers() && !centerIsOurNode) { printAt(X(0.5), Y(0.5) - (getFont().lineHeight() / 2), "Node positions", CENTER, MIDDLE); printAt(X(0.5), Y(0.5) + (getFont().lineHeight() / 2), "will appear here", CENTER, MIDDLE); return; } + // Determine the metersToPx needed to fit all nodes on screen. + getMapSize(&widthMeters, &heightMeters); + calculateMapScale(); // metersToPx = fit-all-nodes scale + const float metersToPxFit = metersToPx; + + // Pick the highest zoom whose native scale fits all nodes (no downsampling, no dither noise). + { + const float R = 6378137.0f; + const float latRad = latCenter * DEG_TO_RAD; + + // Collect unique zooms, sort descending (highest detail first) + int zooms[16] = {}; + int nzooms = 0; + for (int i = 0; i < map_tile_count && nzooms < 16; i++) { + bool found = false; + for (int j = 0; j < nzooms; j++) { + if (zooms[j] == map_tile_zooms[i]) { + found = true; + break; + } + } + if (!found) + zooms[nzooms++] = map_tile_zooms[i]; + } + for (int i = 0; i < nzooms - 1; i++) { + for (int j = i + 1; j < nzooms; j++) { + if (zooms[j] > zooms[i]) { + int t = zooms[i]; + zooms[i] = zooms[j]; + zooms[j] = t; + } + } + } + + int chosenZoom = (nzooms > 0) ? zooms[nzooms - 1] : 13; // fallback: widest zoom + float chosenMetersToPx = metersToPxFit; // fallback: fit-scale (may downsample) + + if (s_zoomLocked && s_lockedZoom >= 0) { + // Use locked zoom at native 1:1 scale — never zoom out for new nodes + chosenZoom = s_lockedZoom; + float mpp = (2.0f * M_PI * R / (256.0f * (float)(1 << chosenZoom))) * cosf(latRad); + chosenMetersToPx = 1.0f / mpp; + } else if ((markers.empty() || metersToPxFit <= 0.0f) && nzooms > 0) { + // No spread to fit (own node only, or single remote node at map center). Use highest zoom at native scale. + chosenZoom = zooms[0]; + float mpp = (2.0f * M_PI * R / (256.0f * (float)(1 << chosenZoom))) * cosf(latRad); + chosenMetersToPx = 1.0f / mpp; + } else { + for (int zi = 0; zi < nzooms; zi++) { + float mpp = (2.0f * M_PI * R / (256.0f * (float)(1 << zooms[zi]))) * cosf(latRad); + float nativeMetersToPx = 1.0f / mpp; + if (nativeMetersToPx <= metersToPxFit) { + // This zoom at native scale shows all nodes — use it (highest detail that fits) + chosenZoom = zooms[zi]; + chosenMetersToPx = nativeMetersToPx; + break; + } + } + } + + if (!s_zoomLocked) + s_autoFitZoom = chosenZoom; + metersToPx = chosenMetersToPx; + s_lastRenderedZoom = chosenZoom; + drawMapTileBackground(chosenZoom); + + char zoomLabel[8]; + snprintf(zoomLabel, sizeof(zoomLabel), "z%d", chosenZoom); + int16_t zoomLabelW = getTextWidth(zoomLabel); + int16_t zoomLabelH = getFont().lineHeight(); + int16_t zoomLabelX = width() - zoomLabelW - 3; + int16_t zoomLabelY = 2; + fillRect(zoomLabelX - 2, zoomLabelY - 1, zoomLabelW + 4, zoomLabelH + 2, WHITE); + printAt(zoomLabelX, zoomLabelY, zoomLabel, LEFT, TOP); + } + // Helper: draw rounded rectangle centered at x,y auto fillRoundedRect = [&](int16_t cx, int16_t cy, int16_t w, int16_t h, int16_t r, uint16_t color) { int16_t x = cx - (w / 2); @@ -30,16 +379,10 @@ void InkHUD::MapApplet::onRender(bool full) fillCircle(x + w - r - 1, y + h - r - 1, r, color); }; - // Find center of map - getMapCenter(&latCenter, &lngCenter); - calculateAllMarkers(); - getMapSize(&widthMeters, &heightMeters); - calculateMapScale(); - // Draw all markers first for (Marker m : markers) { - int16_t x = X(0.5) + (m.eastMeters * metersToPx); - int16_t y = Y(0.5) - (m.northMeters * metersToPx); + int16_t x = X(0.5) + (int16_t)(m.eastMeters * metersToPx); + int16_t y = Y(0.5) - (int16_t)(m.northMeters * metersToPx); // Add white halo outline first constexpr int outlinePad = 1; @@ -57,10 +400,8 @@ void InkHUD::MapApplet::onRender(bool full) setTextColor(WHITE); // Draw actual marker on top - if (m.hasHopsAway && m.hopsAway > config.lora.hop_limit) { + if (m.hopsAway > config.lora.hop_limit) { printAt(x + 1, y + 1, "X", CENTER, MIDDLE); - } else if (!m.hasHopsAway) { - printAt(x + 1, y + 1, "?", CENTER, MIDDLE); } else { char hopStr[4]; snprintf(hopStr, sizeof(hopStr), "%d", m.hopsAway); @@ -73,6 +414,8 @@ void InkHUD::MapApplet::onRender(bool full) } // Dual map scale bars + if (metersToPx <= 0.0f) + return; int16_t horizPx = width() * 0.25f; int16_t vertPx = height() * 0.25f; float horizMeters = horizPx / metersToPx; @@ -136,11 +479,11 @@ void InkHUD::MapApplet::onRender(bool full) printAt(vertBarX + (bottomLabelW / 2) + 1, bottomLabelY + (bottomLabelH / 2), vertBottomLabel, CENTER, MIDDLE); // Draw our node LAST with full white fill + outline - const meshtastic_NodeInfoLite *ourNode = nodeDB->getMeshNode(nodeDB->getNodeNum()); - meshtastic_PositionLite ourSelfPos; - if (ourNode && nodeDB->hasValidPosition(ourNode) && nodeDB->copyNodePosition(ourNode->num, ourSelfPos)) { - Marker self = calculateMarker(ourSelfPos.latitude_i * 1e-7, ourSelfPos.longitude_i * 1e-7, false, 0); - + if (centerIsOurNode) { + const meshtastic_NodeInfoLite *ourNode = nodeDB->getMeshNode(nodeDB->getNodeNum()); + meshtastic_PositionLite ourSelfPos; + nodeDB->copyNodePosition(ourNode->num, ourSelfPos); + Marker self = calculateMarker(ourSelfPos.latitude_i * 1e-7, ourSelfPos.longitude_i * 1e-7, 0); int16_t centerX = X(0.5) + (self.eastMeters * metersToPx); int16_t centerY = Y(0.5) - (self.northMeters * metersToPx); @@ -174,7 +517,9 @@ void InkHUD::MapApplet::getMapCenter(float *lat, float *lng) if (ourNode && nodeDB->hasValidPosition(ourNode) && nodeDB->copyNodePosition(ourNode->num, ourSelfPos)) { *lat = ourSelfPos.latitude_i * 1e-7; *lng = ourSelfPos.longitude_i * 1e-7; + centerIsOurNode = true; } else { + centerIsOurNode = false; // Find mean lat long coords // ============================ // - assigning X, Y and Z values to position on Earth's surface in 3D space, relative to center of planet @@ -225,6 +570,8 @@ void InkHUD::MapApplet::getMapCenter(float *lat, float *lng) } // All NodeDB processed, find mean values + if (positionCount == 0) + return; xAvg /= positionCount; yAvg /= positionCount; zAvg /= positionCount; @@ -278,18 +625,43 @@ void InkHUD::MapApplet::getMapCenter(float *lat, float *lng) latCenter = *lat; lngCenter = *lng; - // ---------------------------------------------- - // This has given us either: - // - our actual position (preferred), or - // - a mean position (fallback if we had no fix) - // - // What we actually want is to place our center so that our outermost nodes - // end up on the border of our map. The only real use of our "center" is to give - // us a reference frame: which direction is east, and which is west. - //------------------------------------------------ + // When zoom is locked, keep center exactly on own node / zero-hop centroid. + // Skip bounding-box shift so new distant nodes don't move the zoomed view. + if (s_zoomLocked) { + // Own node has no position — re-center on zero-hop centroid instead. + if (!centerIsOurNode) { + uint32_t count = 0; + float xAvg = 0, yAvg = 0, zAvg = 0; + for (uint32_t i = 0; i < nodeDB->getNumMeshNodes(); i++) { + meshtastic_NodeInfoLite *node = nodeDB->getMeshNodeByIndex(i); + if (!nodeDB->hasValidPosition(node) || !shouldDrawNode(node)) + continue; + if (!node->has_hops_away || node->hops_away != 0) + continue; + meshtastic_PositionLite pos; + if (!nodeDB->copyNodePosition(node->num, pos)) + continue; + float latRad2 = pos.latitude_i * 1e-7 * DEG_TO_RAD; + float lngRad2 = pos.longitude_i * 1e-7 * DEG_TO_RAD; + xAvg += cosf(latRad2) * cosf(lngRad2); + yAvg += cosf(latRad2) * sinf(lngRad2); + zAvg += sinf(latRad2); + count++; + } + if (count > 0) { + xAvg /= count; + yAvg /= count; + zAvg /= count; + *lng = atan2f(yAvg, xAvg) * RAD_TO_DEG; + *lat = atan2f(zAvg, sqrtf(xAvg * xAvg + yAvg * yAvg)) * RAD_TO_DEG; + latCenter = *lat; + lngCenter = *lng; + } + } + return; // Do not shift center based on bounding box + } - // Find furthest nodes from our center - // ======================================== + // Find furthest nodes from our center, shift center to midpoint of bounding box float northernmost = latCenter; float southernmost = latCenter; float easternmost = lngCenter; @@ -298,11 +670,8 @@ void InkHUD::MapApplet::getMapCenter(float *lat, float *lng) for (size_t i = 0; i < nodeDB->getNumMeshNodes(); i++) { meshtastic_NodeInfoLite *node = nodeDB->getMeshNodeByIndex(i); - // Skip if no position if (!nodeDB->hasValidPosition(node)) continue; - - // Skip if derived applet doesn't want to show this node on the map if (!shouldDrawNode(node)) continue; @@ -310,15 +679,14 @@ void InkHUD::MapApplet::getMapCenter(float *lat, float *lng) if (!nodeDB->copyNodePosition(node->num, pos)) continue; - // Check for a new top or bottom latitude float latNode = pos.latitude_i * 1e-7; + float lngNode = pos.longitude_i * 1e-7; + northernmost = max(northernmost, latNode); southernmost = min(southernmost, latNode); - // Longitude is trickier - float lngNode = pos.longitude_i * 1e-7; - float degEastward = fmod(((lngNode - lngCenter) + 360), 360); // Degrees traveled east from lngCenter to reach node - float degWestward = abs(fmod(((lngNode - lngCenter) - 360), 360)); // Degrees traveled west from lngCenter to reach node + float degEastward = fmod(((lngNode - lngCenter) + 360), 360); // Degrees east from center to node + float degWestward = abs(fmod(((lngNode - lngCenter) - 360), 360)); // Degrees west from center to node if (degEastward < degWestward) easternmost = max(easternmost, lngCenter + degEastward); else @@ -356,7 +724,7 @@ void InkHUD::MapApplet::getMapSize(uint32_t *widthMeters, uint32_t *heightMeters // Convert and store info we need for drawing a marker // Lat / long to "meters relative to map center", for position on screen // Info about hopsAway, for marker size -InkHUD::MapApplet::Marker InkHUD::MapApplet::calculateMarker(float lat, float lng, bool hasHopsAway, uint8_t hopsAway) +InkHUD::MapApplet::Marker InkHUD::MapApplet::calculateMarker(float lat, float lng, uint8_t hopsAway) { assert(lat != 0 || lng != 0); // Not null island. Applets should check this before calling. @@ -369,11 +737,9 @@ InkHUD::MapApplet::Marker InkHUD::MapApplet::calculateMarker(float lat, float ln float northMeters = cos(bearingFromCenter) * distanceFromCenter; float eastMeters = sin(bearingFromCenter) * distanceFromCenter; - // Store this as a new marker Marker m; m.eastMeters = eastMeters; m.northMeters = northMeters; - m.hasHopsAway = hasHopsAway; m.hopsAway = hopsAway; return m; } @@ -385,11 +751,7 @@ void InkHUD::MapApplet::drawLabeledMarker(meshtastic_NodeInfoLite *node) meshtastic_PositionLite pos; const bool hasPos = nodeDB->copyNodePosition(node->num, pos); assert(hasPos); - Marker m = calculateMarker(pos.latitude_i * 1e-7, // Lat, converted from Meshtastic's internal int32 style - pos.longitude_i * 1e-7, // Long, converted from Meshtastic's internal int32 style - node->has_hops_away, // Is the hopsAway number valid - node->hops_away // Hops away - ); + Marker m = calculateMarker(pos.latitude_i * 1e-7, pos.longitude_i * 1e-7, node->hops_away); // Convert to pixel coords int16_t markerX = X(0.5) + (m.eastMeters * metersToPx); @@ -412,8 +774,6 @@ void InkHUD::MapApplet::drawLabeledMarker(meshtastic_NodeInfoLite *node) uint8_t markerSize; bool tooManyHops = node->hops_away > config.lora.hop_limit; - bool isOurNode = node->num == nodeDB->getNodeNum(); - bool unknownHops = !node->has_hops_away && !isOurNode; // Parse any non-ascii chars in the short name, // and use last 4 instead if unknown / can't render @@ -426,8 +786,6 @@ void InkHUD::MapApplet::drawLabeledMarker(meshtastic_NodeInfoLite *node) // Pick emblem style if (tooManyHops) markerSize = getTextWidth("!"); - else if (unknownHops) - markerSize = markerSizeMin; else markerSize = map(node->hops_away, 0, config.lora.hop_limit, markerSizeMax, markerSizeMin); @@ -482,27 +840,18 @@ void InkHUD::MapApplet::drawLabeledMarker(meshtastic_NodeInfoLite *node) if (tooManyHops) printAt(markerX, markerY, "!", CENTER, MIDDLE); else - drawCross(markerX, markerY, markerSize); // The fewer the hops, the larger the marker. Also handles unknownHops + drawCross(markerX, markerY, markerSize); } // Check if we actually have enough nodes which would be shown on the map -// Need at least two, to draw a sensible map bool InkHUD::MapApplet::enoughMarkers() { - size_t count = 0; for (size_t i = 0; i < nodeDB->getNumMeshNodes(); i++) { meshtastic_NodeInfoLite *node = nodeDB->getMeshNodeByIndex(i); - - // Count nodes if (nodeDB->hasValidPosition(node) && shouldDrawNode(node)) - count++; - - // We need to find two - if (count == 2) - return true; // Two nodes is enough for a sensible map + return true; } - - return false; // No nodes would be drawn (or just the one, uselessly at 0,0) + return false; } // Calculate how far north and east of map center each node is @@ -529,36 +878,30 @@ void InkHUD::MapApplet::calculateAllMarkers() if (node->num == nodeDB->getNodeNum()) continue; + // Skip nodes with unknown hop count — partial info, not useful to plot + if (!node->has_hops_away) + continue; + meshtastic_PositionLite pos; if (!nodeDB->copyNodePosition(node->num, pos)) continue; - // Calculate marker and store it - markers.push_back(calculateMarker(pos.latitude_i * 1e-7, // Lat, converted from Meshtastic's internal int32 style - pos.longitude_i * 1e-7, // Long, converted from Meshtastic's internal int32 style - node->has_hops_away, // Is the hopsAway number valid - node->hops_away // Hops away - )); + markers.push_back(calculateMarker(pos.latitude_i * 1e-7, pos.longitude_i * 1e-7, node->hops_away)); } } -// Determine the conversion factor between metres, and pixels on screen -// May be overridden by derived applet, if custom scale required (fixed map size?) void InkHUD::MapApplet::calculateMapScale() { - // Aspect ratio of map and screen - // - larger = wide, smaller = tall - // - used to set scale, so that widest map dimension fits in applet + if (widthMeters == 0 || heightMeters == 0) { + metersToPx = 0; + return; + } float mapAspectRatio = (float)widthMeters / heightMeters; float appletAspectRatio = (float)width() / height(); - - // "Shrink to fit" - // Scale the map so that the largest dimension is fully displayed - // Because aspect ratio will be maintained, the other dimension will appear "padded" if (mapAspectRatio > appletAspectRatio) - metersToPx = (float)width() / widthMeters; // Too wide for applet. Constrain to fit width. + metersToPx = (float)width() / widthMeters; else - metersToPx = (float)height() / heightMeters; // Too tall for applet. Constrain to fit height. + metersToPx = (float)height() / heightMeters; } // Draw an x, centered on a specific point diff --git a/src/graphics/niche/InkHUD/Applets/Bases/Map/MapApplet.h b/src/graphics/niche/InkHUD/Applets/Bases/Map/MapApplet.h index 11dfb39d9..fe1ff8d7a 100644 --- a/src/graphics/niche/InkHUD/Applets/Bases/Map/MapApplet.h +++ b/src/graphics/niche/InkHUD/Applets/Bases/Map/MapApplet.h @@ -15,10 +15,13 @@ The base applet doesn't handle any events; this is left to the derived applets. #pragma once #include "configuration.h" +#include #include "graphics/niche/InkHUD/Applet.h" +#include "GPSStatus.h" #include "MeshModule.h" +#include "Observer.h" #include "gps/GeoCoord.h" namespace NicheGraphics::InkHUD @@ -27,33 +30,55 @@ namespace NicheGraphics::InkHUD class MapApplet : public Applet { public: + MapApplet(); void onRender(bool full) override; + MapApplet *asMapApplet() override { return this; } // Identify as MapApplet without RTTI + + // Zoom lock — shared across all MapApplet instances (static) + static constexpr int ZOOM_MAX_NO_TILES = 16; + + void zoomIn(); + void zoomOut(); + void resetZoom(); + bool isZoomLocked() const { return s_zoomLocked; } + bool canZoomIn() const; + bool canZoomOut() const; + protected: virtual bool shouldDrawNode(meshtastic_NodeInfoLite *node) { return true; } // Allow derived applets to filter the nodes virtual void getMapCenter(float *lat, float *lng); virtual void getMapSize(uint32_t *widthMeters, uint32_t *heightMeters); - bool enoughMarkers(); // Anything to draw? + virtual bool enoughMarkers(); // Anything to draw? void drawLabeledMarker(meshtastic_NodeInfoLite *node); // Highlight a specific marker private: + int onGpsStatusUpdate(const meshtastic::Status *status); + CallbackObserver gpsStatusObserver = + CallbackObserver(this, &MapApplet::onGpsStatusUpdate); + + static bool s_zoomLocked; + static int s_lockedZoom; + static int s_lastRenderedZoom; + static int s_autoFitZoom; // Zoom chosen by auto-fit (updated whenever not locked) // Position and size of a marker to be drawn struct Marker { float eastMeters = 0; // Meters east of map center. Negative if west. float northMeters = 0; // Meters north of map center. Negative if south. - bool hasHopsAway = false; - uint8_t hopsAway = 0; // Determines marker size + uint8_t hopsAway = 0; // Determines marker size }; - Marker calculateMarker(float lat, float lng, bool hasHopsAway, uint8_t hopsAway); + Marker calculateMarker(float lat, float lng, uint8_t hopsAway); void calculateAllMarkers(); void calculateMapScale(); // Conversion factor for meters to pixels + void drawMapTileBackground(int zoom); // Draw georeferenced tile at zoom void drawCross(int16_t x, int16_t y, uint8_t size); // Draw the X used for most markers - float metersToPx = 0; // Conversion factor for meters to pixels - float latCenter = 0; // Map center: latitude - float lngCenter = 0; // Map center: longitude + float metersToPx = 0; // Conversion factor for meters to pixels + float latCenter = 0; // Map center: latitude + float lngCenter = 0; // Map center: longitude + bool centerIsOurNode = false; // True if map is centered on our own position (GPS or phone) std::list markers; uint32_t widthMeters = 0; // Map width: meters diff --git a/src/graphics/niche/InkHUD/Applets/Bases/Map/MapTile.h b/src/graphics/niche/InkHUD/Applets/Bases/Map/MapTile.h new file mode 100644 index 000000000..fee0f60f9 --- /dev/null +++ b/src/graphics/niche/InkHUD/Applets/Bases/Map/MapTile.h @@ -0,0 +1,9 @@ +#pragma once +#include + +static const int map_tile_count = 0; +static const int map_tile_zooms[] = {}; +static const int map_tile_tx[] = {}; +static const int map_tile_ty[] = {}; +static const int map_tile_sizes[] = {}; +static const uint8_t *map_tile_data[] = {}; diff --git a/src/graphics/niche/InkHUD/Applets/System/Menu/MenuAction.h b/src/graphics/niche/InkHUD/Applets/System/Menu/MenuAction.h index 4c70c434f..e3011b0e2 100644 --- a/src/graphics/niche/InkHUD/Applets/System/Menu/MenuAction.h +++ b/src/graphics/niche/InkHUD/Applets/System/Menu/MenuAction.h @@ -130,6 +130,10 @@ enum MenuAction { RESET_NODEDB_ALL, RESET_NODEDB_KEEP_FAVORITES, WIPE_MESSAGES_ALL, + // Map zoom (MapApplet and FavoritesMapApplet) + MAP_ZOOM_IN, + MAP_ZOOM_OUT, + MAP_ZOOM_RESET, }; } // namespace NicheGraphics::InkHUD diff --git a/src/graphics/niche/InkHUD/Applets/System/Menu/MenuApplet.cpp b/src/graphics/niche/InkHUD/Applets/System/Menu/MenuApplet.cpp index 288dc0037..2024f17c6 100644 --- a/src/graphics/niche/InkHUD/Applets/System/Menu/MenuApplet.cpp +++ b/src/graphics/niche/InkHUD/Applets/System/Menu/MenuApplet.cpp @@ -10,6 +10,7 @@ #include "RTC.h" #include "Router.h" #include "airtime.h" +#include "graphics/niche/InkHUD/Applets/Bases/Map/MapApplet.h" #include "graphics/niche/Utils/FlashData.h" #include "main.h" #include "mesh/generated/meshtastic/deviceonly.pb.h" @@ -1021,6 +1022,27 @@ void InkHUD::MenuApplet::execute(MenuItem item) inkhud->forceUpdate(Drivers::EInk::UpdateTypes::FULL, true); break; + case MAP_ZOOM_IN: { + MapApplet *mapApplet = borrowedTileOwner ? borrowedTileOwner->asMapApplet() : nullptr; + if (mapApplet) + mapApplet->zoomIn(); + break; + } + + case MAP_ZOOM_OUT: { + MapApplet *mapApplet = borrowedTileOwner ? borrowedTileOwner->asMapApplet() : nullptr; + if (mapApplet) + mapApplet->zoomOut(); + break; + } + + case MAP_ZOOM_RESET: { + MapApplet *mapApplet = borrowedTileOwner ? borrowedTileOwner->asMapApplet() : nullptr; + if (mapApplet) + mapApplet->resetZoom(); + break; + } + default: LOG_WARN("Action not implemented"); } @@ -1046,6 +1068,20 @@ void InkHUD::MenuApplet::showPage(MenuPage page) items.push_back(MenuItem("Next Tile", MenuAction::NEXT_TILE, MenuPage::ROOT)); // Only if multiple applets shown items.push_back(MenuItem("Send", MenuPage::SEND)); + + // Map zoom controls — only when viewing a map applet + { + MapApplet *mapApplet = borrowedTileOwner ? borrowedTileOwner->asMapApplet() : nullptr; + if (mapApplet) { + if (mapApplet->canZoomIn()) + items.push_back(MenuItem("Zoom In", MenuAction::MAP_ZOOM_IN, MenuPage::EXIT)); + if (mapApplet->canZoomOut()) + items.push_back(MenuItem("Zoom Out", MenuAction::MAP_ZOOM_OUT, MenuPage::EXIT)); + if (mapApplet->isZoomLocked()) + items.push_back(MenuItem("Reset Zoom", MenuAction::MAP_ZOOM_RESET, MenuPage::EXIT)); + } + } + items.push_back(MenuItem("Options", MenuPage::OPTIONS)); // items.push_back(MenuItem("Display Off", MenuPage::EXIT)); // TODO items.push_back(MenuItem("Node Config", MenuPage::NODE_CONFIG)); diff --git a/src/graphics/niche/InkHUD/Applets/User/FavoritesMap/FavoritesMapApplet.cpp b/src/graphics/niche/InkHUD/Applets/User/FavoritesMap/FavoritesMapApplet.cpp index 290546a6f..8002219d5 100644 --- a/src/graphics/niche/InkHUD/Applets/User/FavoritesMap/FavoritesMapApplet.cpp +++ b/src/graphics/niche/InkHUD/Applets/User/FavoritesMap/FavoritesMapApplet.cpp @@ -2,6 +2,7 @@ #include "./FavoritesMapApplet.h" #include "NodeDB.h" +#include "configuration.h" using namespace NicheGraphics; @@ -11,6 +12,15 @@ bool InkHUD::FavoritesMapApplet::shouldDrawNode(meshtastic_NodeInfoLite *node) return node && (node->num == nodeDB->getNodeNum() || nodeInfoLiteIsFavorite(node)); } +// Show map as long as our own node has a position, even with no favorites yet. +bool InkHUD::FavoritesMapApplet::enoughMarkers() +{ + const meshtastic_NodeInfoLite *ourNode = nodeDB->getMeshNode(nodeDB->getNodeNum()); + if (ourNode && nodeDB->hasValidPosition(ourNode)) + return true; + return MapApplet::enoughMarkers(); +} + void InkHUD::FavoritesMapApplet::onRender(bool full) { // Custom empty state text for favorites-only map. diff --git a/src/graphics/niche/InkHUD/Applets/User/FavoritesMap/FavoritesMapApplet.h b/src/graphics/niche/InkHUD/Applets/User/FavoritesMap/FavoritesMapApplet.h index da5fb0dc3..3f6d3381a 100644 --- a/src/graphics/niche/InkHUD/Applets/User/FavoritesMap/FavoritesMapApplet.h +++ b/src/graphics/niche/InkHUD/Applets/User/FavoritesMap/FavoritesMapApplet.h @@ -27,7 +27,10 @@ class FavoritesMapApplet : public MapApplet, public SinglePortModule void onRender(bool full) override; protected: + void onActivate() override { loopbackOk = true; } + void onDeactivate() override { loopbackOk = false; } bool shouldDrawNode(meshtastic_NodeInfoLite *node) override; + bool enoughMarkers() override; ProcessMessage handleReceived(const meshtastic_MeshPacket &mp) override; NodeNum lastFrom = 0; // Sender of most recent favorited (non-local) position packet diff --git a/src/graphics/niche/InkHUD/Applets/User/Positions/PositionsApplet.h b/src/graphics/niche/InkHUD/Applets/User/Positions/PositionsApplet.h index d0d3e5f07..5cb20ed3b 100644 --- a/src/graphics/niche/InkHUD/Applets/User/Positions/PositionsApplet.h +++ b/src/graphics/niche/InkHUD/Applets/User/Positions/PositionsApplet.h @@ -27,6 +27,8 @@ class PositionsApplet : public MapApplet, public SinglePortModule void onRender(bool full) override; protected: + void onActivate() override { loopbackOk = true; } + void onDeactivate() override { loopbackOk = false; } ProcessMessage handleReceived(const meshtastic_MeshPacket &mp) override; NodeNum lastFrom = 0; // Sender of most recent (non-local) position packet diff --git a/src/graphics/niche/InkHUD/docs/README.md b/src/graphics/niche/InkHUD/docs/README.md index 2ebff37de..817b76c3f 100644 --- a/src/graphics/niche/InkHUD/docs/README.md +++ b/src/graphics/niche/InkHUD/docs/README.md @@ -254,6 +254,20 @@ You will need to add these lines to any variants which will use your applet. If you need to create several similar applets, it might make sense to create a reusable base class. Several of these already exist in `src/graphics/niche/InkHUD/Applets/Bases`, but use these with caution, as they may be modified in future. +#### Map Applet Base + +`MapApplet` (`src/graphics/niche/InkHUD/Applets/Bases/Map/MapApplet.h`) is a base class for applets that plot node positions on a map. It handles tile rendering, zoom control, scale bars, and GPS tracking. `PositionsApplet` and `FavoritesMapApplet` both inherit from it. + +##### Map Tiles + +Map tiles are stored in `MapTile.h` (`src/graphics/niche/InkHUD/Applets/Bases/Map/MapTile.h`). The file committed to the repository contains no tile data by default — the map applets work without tiles, falling back to the original marker-only display. + +Tiles are 256×256 pixels, 1-bit (column-major bit packing), compressed per tile with LZ4 to keep flash usage low. + +##### Zoom Controls + +When the menu is opened from a map applet, zoom controls appear automatically. Zoom In and Zoom Out step through the available tile zoom levels. Reset Zoom returns to the default auto-fit behavior, where the map scales to show all visible nodes. + #### System Applets So far, we have been talking about "user applets". We also recognize a separate category of "system applets". These handle things like the menu, and the boot screen. These often need special handling, and need to be implemented manually. From 0baf8b1db6f6c5e2fe9c08b7da8f244cc0d01ba4 Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Sun, 28 Jun 2026 06:50:35 -0500 Subject: [PATCH 45/86] Portduino WASM hello world (still experimental) (#10793) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add ARCH_PORTDUINO_WASM build: meshtasticd in WebAssembly over WebUSB Compile the full portduino firmware to WebAssembly (Emscripten) so a real node runs in a browser tab or headless Node, driving a LoRa radio over WebUSB through a CH341 — the desktop Ch341Hal path with its libusb backend swapped for a WebUSB one. The native/desktop portduino build is unchanged. New build env under src/platform/portduino/wasm/ (excluded from the native build_src_filter): WebUSB libpinedio backend, config/FS/region/MAC/PhoneAPI glue, wasm setup/loop, JS WebUSB runtime, and build stubs. bin/build-portduino-wasm.sh runs a standalone cached emcc build to build/wasm/meshnode.{mjs,wasm}. Six firmware sources gain #ifdef ARCH_PORTDUINO_WASM guards (single-threaded cooperative emscripten_sleep loop, continuous RX, US region default, std RNG, no-popen exec); none affect non-wasm builds. * PortDuino WASM: CI build job, cross-platform libdeps, configurable adapter bin/build-portduino-wasm.sh: auto-detect native-macos (macOS) vs native (Linux/CI) libdeps with a NATIVE_ENV override, and use the SAME env's Crypto with an XEdDSA-present guard. Drops the heltec-v3/Crypto borrow — the meshtastic/Crypto pin already ships XEdDSA; the native libdeps cache was just stale. No env pin change. Add .github/workflows/build_portduino_wasm.yml: build the ARCH_PORTDUINO_WASM target in CI (ubuntu + emsdk, native libdeps) so it can't silently bit-rot; asserts build/wasm/meshnode.{mjs,wasm}. src/platform/portduino/wasm: add wasm_set_lora_* setters so the JS host can configure any CH341 LoRa adapter (module, USB ids, DIO/TCXO, SPI speed, pins); wasm_config_apply falls back to the MeshToad defaults when unset. * WASM: build as a first-class [env:wasm] via platform-wasm (retire emcc script) Replace the standalone emcc build (bin/build-portduino-wasm.sh) with a normal PlatformIO env, `pio run -e wasm`, using the new meshtastic/platform-wasm platform (emcc/em++ + Asyncify, the WASM sibling of platform-native). The portduino WebAssembly node is now built the same way as every other target. - variants/native/portduino/platformio.ini: add [env:wasm] (platform pinned to platform-wasm, board wasm). Translates the script's source set into a curated build_src_filter, the ~30 EXCLUDE_* defines, and the lib set; defines ARCH_PORTDUINO_WASM in-repo so correctness doesn't hinge on the platform's board.json. - extra_scripts/wasm_link_flags.py: the firmware-specific emcc *link* settings (EXPORT_NAME, EXPORTED_RUNTIME_METHODS, EXPORTED_FUNCTIONS, ASYNCIFY_IMPORTS). PlatformIO feeds build_flags to compile only, so these must ride LINKFLAGS; without it the WebUSB Asyncify seam and the JS host's runtime methods are dropped (the _wasm_* exports survive only via EMSCRIPTEN_KEEPALIVE). - PortduinoGlue.{h,cpp}: guard the yaml-cpp dependency out of the WASM build (#ifndef ARCH_PORTDUINO_WASM around the include, emit_yaml/loadConfig/ readGPIOFromYaml). The browser node configures via the wasm_set_lora_* setters and dead-strips the YAML path; this drops the host yaml-cpp build dependency entirely. Native is unchanged (guards are inert there). - portduino_glue_wasm.cpp / portduino_main_wasm.cpp: repair EM_ASM JS that a formatter had mangled (!== -> != =, regex split) in the prior landing; the emcc link succeeds regardless, so CI now runs `node --check meshnode.mjs`. - .github/workflows/build_portduino_wasm.yml: build via `pio run -e wasm` (artifacts under .pio/build/wasm/), trigger on the shared inputs the env inherits (root platformio.ini, bin/platformio-*.py). - NodeDB.cpp: drop the dead ARCH_PORTDUINO_WASM region-default branch (region now defaults the same as native). - Crypto renovate pins: add the missing gitBranch so they track upstream. Output: .pio/build/wasm/meshnode.{mjs,wasm} (ES module, factory createMeshNode). Verified: pio run -e wasm (against the published platform archive), node --check, module instantiates in Node with all exports; native-macos + Docker native unit tests (450/450) still pass. * Fix name on the Piggystick * wasm: pin platform-wasm at the GPL-3.0-relicensed commit platform-wasm's LICENSE was always GPLv3 (matching this firmware), but its platform.json/README still declared Apache-2.0 (mis-copied from platform-native). That's fixed upstream in b83fa5b; bump the [env:wasm] pin to it. Build output is unchanged (license metadata only). Verified: pio run -e wasm against b83fa5b. * wasm: make reboot() actually restart the node (was a no-op) In wasm the reboot path is live (main.cpp -> Power::powerCommandsCheck -> Power::reboot), but Power::reboot's ARCH_PORTDUINO arm tore down SPI/Wire/Serial and then called the no-op ::reboot() stub — leaving the node running with a dead radio until the tab was manually reloaded. Triggers include an admin/phone reboot, factory reset, the "reconfigure failed" path, and the 60 s stuck-TX hardware watchdog (RadioLibInterface). - Power::reboot(): add an ARCH_PORTDUINO_WASM arm (before ARCH_PORTDUINO, since the wasm build defines both) that skips the host teardown and just calls ::reboot(). notifyReboot already let modules persist. - ::reboot() (glue): hand off to the JS host — browser reloads the tab (NodeDB state survives via IDBFS, same identity returns); headless calls Module.onReboot if provided, else logs. Loose !=/== so clang-format doesn't mangle the EM_ASM JS. - README: document the reboot handoff + the Module.onReboot hook. Verified: pio run -e wasm + node --check (EM_ASM intact); native-macos unaffected. * wasm: rename env to native-wasm and run it in the main CI matrix Rename [env:wasm] -> [env:native-wasm] for consistency with the portduino native family (native, native-macos, native-tft). The build dir follows to .pio/build/native-wasm/ (artifact is still meshnode.{mjs,wasm}); the PIOENV guard in extra_scripts/wasm_link_flags.py, the README, and the companion wrapper move with it. The board stays `wasm`. Also wire the build into normal CI: build_portduino_wasm.yml becomes a reusable workflow (workflow_call) invoked as the `build-wasm` job of main_matrix.yml, so the WebAssembly node is built like every other platform instead of on a separate path trigger. * native-wasm: auto-locate the Emscripten SDK (pre-build script) `pio run -e native-wasm` failed with "emcc not found" whenever it was invoked from a shell that hadn't sourced emsdk_env.sh — a VS Code task, an IDE build button, a bare terminal. Add a pre: extra script that probes the usual emsdk locations ($EMSDK_ENV, $EMSDK, ~/emsdk, ./.emsdk, the sibling companion checkout), sources emsdk_env.sh, and imports the resulting environment so the platform builder and emcc see PATH/EMSDK/EM_CONFIG. No-op when emcc is already reachable (CI), silent when no SDK is found (the platform emits its own error). * wasm: address PR review feedback - js/bridge.js: import CH341 from "./ch341.js" (sibling in this layout), not "../src/ch341.js" which doesn't resolve here. - js/ch341.js: a zero-length transferIn while MISO bytes are still outstanding now throws instead of breaking out with a partially-filled buffer — silent SPI corruption becomes a loud error, matching the comment above it. - libpinedio_webusb.c: webusb_set_auto_cs honors the AUTO_CS option (? 1 : 0) instead of the always-on ? 1 : 1. Runtime behavior is unchanged — Ch341Hal sets AUTO_CS=0 right after pinedio_init (RadioLib drives the active-low NSS); the option just isn't set yet at init, so this now correctly defaults off. - SX126xInterface.cpp: the RX-start error log now names the method actually called (startReceive vs startReceiveDutyCycleAuto) instead of hardcoding the duty-cycle name in the WASM branch. * native-wasm: drop the emsdk bootstrap shim (now in platform-wasm) The Emscripten SDK auto-location moved into the platform-wasm builder, so the firmware no longer needs its own pre: extra script. Remove extra_scripts/wasm_emsdk_env.py and bump the platform pin to the build that carries the bootstrap. The wasm_link_flags.py post script stays — those exported fns / runtime methods / Asyncify import seam are firmware-app-specific. * wasm: use the canonical companion name (meshtasticd-wasm-node) The companion repo was renamed meshtastic-web-node -> meshtasticd-wasm-node; fix the stale name in the wasm README and bump the platform pin to the build that promotes the canonical name in its emsdk auto-location. * wasm: re-entrancy guard for the API/region entry points + flaky-open retry Two robustness fixes for the browser node: - Re-entrancy guard. The node is single-threaded + Asyncify: while setup()/loop() is suspended inside a WebUSB transfer, the JS event loop is free, so a stray DOM/timer callback that re-enters a wasm_* entry point starts a second Asyncify unwind ("async operation already in flight" abort) or clobbers shared PhoneAPI state (observed as a "PhoneAPI::available unexpected state" flood). Add a g_wasm_in_firmware flag set around setup()/loop() (portduino_main_wasm.cpp); the wasm_set_region / wasm_api_to_radio / wasm_api_from_radio / wasm_api_available entry points now reject a mid-tick call (return busy) instead of corrupting or aborting. The host must still call them between ticks — this is the safety net the design lacked, not a substitute for the JS queue. - CH341 open retry (js/bridge.js). First-connect WebUSB is flaky — the interface is briefly unclaimable right after the grant, or held by a prior session, giving a transient "Could not open SPI: -1". Retry the open with a short backoff, closing the device between attempts so claimInterface starts clean. * wasm: exclude emscripten-only sources from cppcheck The `check` board matrix runs `pio check` (cppcheck) over all of src/, including src/platform/portduino/wasm/. cppcheck can't parse the EM_ASYNC_JS/ EM_JS macros (Syntax Error: AST broken at libpinedio_webusb.c:39, internalAstError) and these sources are not part of any checked board build ([env:native-wasm] is board_level=extra, compiled by the build-wasm CI job). Suppress the wasm dir in suppressions.txt, the same way generated/ and .pio/ are already excluded. * wasm: coalesce FS.syncfs so two never run at once IDBFS syncfs is async; the explicit wasm_fs_sync (5s timer + post-save + beforeunload) could overlap a prior in-flight sync, warning "2 FS.syncfs operations in flight at once". Serialize: if a sync is running, mark a pending re-sync and let the in-flight one chain it on completion — at most one in flight, trailing writes still flushed. (Companion drops IDBFS autoPersist so this is the single persistence path.) * wasm: silence false-positive SAST on the emscripten glue - extra_scripts/wasm_link_flags.py: restore the trunk-ignore-all(ruff/F821, flake8/F821) header every other SCons extra_script carries; Import/env are SConscript-injected globals, so ruff/flake8 flag them as undefined. - .semgrepignore: exclude src/platform/portduino/wasm/js/ (browser WebUSB glue, not part of the firmware binary). The unsafe-formatstring rule false-positives on its benign retry/diagnostic console logs. * Update .github/workflows/build_portduino_wasm.yml Co-authored-by: Austin --------- Co-authored-by: Austin --- .github/workflows/build_portduino_wasm.yml | 61 +++ .github/workflows/main_matrix.yml | 8 + .gitignore | 8 + .semgrepignore | 4 + bin/config.d/lora-piggystick-lr1121.yaml | 4 +- extra_scripts/wasm_link_flags.py | 41 ++ src/Power.cpp | 8 + src/main.cpp | 16 + src/mesh/HardwareRNG.cpp | 3 + src/mesh/InterfacesTemplates.cpp | 2 + src/mesh/LR11x0Interface.cpp | 4 + src/mesh/SX126xInterface.cpp | 10 +- src/platform/portduino/PortduinoGlue.cpp | 31 ++ src/platform/portduino/PortduinoGlue.h | 9 + src/platform/portduino/wasm/README.md | 82 ++++ .../portduino/wasm/include/libpinedio-usb.h | 79 ++++ src/platform/portduino/wasm/js/bridge.js | 115 +++++ src/platform/portduino/wasm/js/ch341.js | 186 ++++++++ src/platform/portduino/wasm/js/protocol.js | 96 ++++ .../portduino/wasm/libpinedio_webusb.c | 172 +++++++ .../portduino/wasm/portduino_glue_wasm.cpp | 426 ++++++++++++++++++ .../portduino/wasm/portduino_main_wasm.cpp | 67 +++ src/platform/portduino/wasm/stubs/argp.h | 78 ++++ .../portduino/wasm/stubs/serializer_stub.cpp | 18 + suppressions.txt | 5 + variants/native/portduino.ini | 5 +- variants/native/portduino/platformio.ini | 108 +++++ 27 files changed, 1641 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/build_portduino_wasm.yml create mode 100644 extra_scripts/wasm_link_flags.py create mode 100644 src/platform/portduino/wasm/README.md create mode 100644 src/platform/portduino/wasm/include/libpinedio-usb.h create mode 100644 src/platform/portduino/wasm/js/bridge.js create mode 100644 src/platform/portduino/wasm/js/ch341.js create mode 100644 src/platform/portduino/wasm/js/protocol.js create mode 100644 src/platform/portduino/wasm/libpinedio_webusb.c create mode 100644 src/platform/portduino/wasm/portduino_glue_wasm.cpp create mode 100644 src/platform/portduino/wasm/portduino_main_wasm.cpp create mode 100644 src/platform/portduino/wasm/stubs/argp.h create mode 100644 src/platform/portduino/wasm/stubs/serializer_stub.cpp diff --git a/.github/workflows/build_portduino_wasm.yml b/.github/workflows/build_portduino_wasm.yml new file mode 100644 index 000000000..03bc2c1f2 --- /dev/null +++ b/.github/workflows/build_portduino_wasm.yml @@ -0,0 +1,61 @@ +name: Build PortDuino WASM + +# Reusable workflow — called as the `build-wasm` job of the main CI workflow +# (main_matrix.yml), so the WebAssembly portduino node is built like every other +# platform. Builds the [env:native-wasm] target (src/platform/portduino/wasm/) with +# `pio run -e native-wasm` so it can't silently bit-rot. Software/CI only — asserts the +# full mesh stack + RadioLib + Crypto compile and link to meshnode.{mjs,wasm} +# via the meshtastic/platform-wasm PlatformIO platform (emcc/Asyncify). It does +# NOT exercise the radio (that's CH341/WebUSB, hardware). + +on: + workflow_call: + workflow_dispatch: + +permissions: {} + +jobs: + build-wasm: + name: Build PortDuino WASM + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + submodules: recursive + + # Python + PlatformIO. (`pio run -e native-wasm` installs its own libdeps and the + # platform-wasm/framework-portduino packages; it needs no native apt libs.) + - name: Setup native build + uses: ./.github/actions/setup-native + + - name: Setup Emscripten SDK + uses: emscripten-core/setup-emsdk@v16 + with: + version: 6.0.1 + actions-cache-folder: emsdk-cache + + - name: Build PortDuino WASM + run: platformio run -e native-wasm + + - name: Assert WASM artifacts + run: | + OUT=.pio/build/native-wasm + if [ ! -s "$OUT/meshnode.mjs" ] || [ ! -s "$OUT/meshnode.wasm" ]; then + echo "::error::wasm artifacts missing"; exit 1 + fi + # The emcc link succeeds even if embedded EM_ASM JS is malformed, so + # parse the generated ES-module loader to catch runtime-JS breakage + # (e.g. a formatter splitting !== inside EM_ASM). + node --check "$OUT/meshnode.mjs" || { echo "::error::meshnode.mjs failed JS parse"; exit 1; } + ls -lah "$OUT"/meshnode.* + + - name: Upload WASM artifacts + if: success() + uses: actions/upload-artifact@v7 + with: + name: portduino-wasm + overwrite: true + path: | + .pio/build/native-wasm/meshnode.mjs + .pio/build/native-wasm/meshnode.wasm + retention-days: 14 diff --git a/.github/workflows/main_matrix.yml b/.github/workflows/main_matrix.yml index e5c75d2cc..73420a716 100644 --- a/.github/workflows/main_matrix.yml +++ b/.github/workflows/main_matrix.yml @@ -146,6 +146,14 @@ jobs: checks: write uses: ./.github/workflows/test_native.yml + build-wasm: + # Build the WebAssembly portduino node ([env:native-wasm]) as part of normal CI, + # like the other platforms. It's a dedicated job (not a row in the `build` + # matrix) because its artifact is meshnode.{mjs,wasm} — not a flashable + # .bin/.uf2/.hex — and it needs the Emscripten SDK; board_level=extra keeps + # it out of generate_ci_matrix.py. + uses: ./.github/workflows/build_portduino_wasm.yml + docker: permissions: # Needed for pushing to GHCR. contents: read diff --git a/.gitignore b/.gitignore index 55e90a8f2..bee3fe4c3 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,11 @@ userPrefs.jsonc.mcp-session-bak # compiled .proto outputs are ephemeral build artifacts. build/fixtures/ bin/_generated/ + +# portduino WASM build (pio run -e native-wasm) + Emscripten SDK. The artifacts land in +# .pio/build/native-wasm/ (already ignored via .pio/); these cover the retired +# standalone emcc build's output dirs and any in-repo emsdk checkout. +build/wasm/ +build/wasm-obj/ +build/wasm-*.log +.emsdk/ diff --git a/.semgrepignore b/.semgrepignore index b4267ad23..fff35f6b7 100644 --- a/.semgrepignore +++ b/.semgrepignore @@ -1,2 +1,6 @@ .github/workflows/main_matrix.yml src/mesh/compression/unishox2.cpp +# Emscripten/WebUSB browser glue for the wasm node — not part of the firmware +# binary or its security surface. The format-string rule false-positives on its +# benign retry/diagnostic console logs. +src/platform/portduino/wasm/js/ diff --git a/bin/config.d/lora-piggystick-lr1121.yaml b/bin/config.d/lora-piggystick-lr1121.yaml index e11c78dd3..d6b88198b 100644 --- a/bin/config.d/lora-piggystick-lr1121.yaml +++ b/bin/config.d/lora-piggystick-lr1121.yaml @@ -1,5 +1,5 @@ Meta: - name: Lora Meshstick SX1262 + name: PiggyStick LR1121 support: community compatible: - usb @@ -12,6 +12,6 @@ Lora: Busy: 4 spidev: ch341 DIO3_TCXO_VOLTAGE: 1.8 -# USB_Serialnum: 12345678 + # USB_Serialnum: 12345678 USB_PID: 0x5512 USB_VID: 0x1A86 diff --git a/extra_scripts/wasm_link_flags.py b/extra_scripts/wasm_link_flags.py new file mode 100644 index 000000000..4b7747364 --- /dev/null +++ b/extra_scripts/wasm_link_flags.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# trunk-ignore-all(ruff/F821) +# trunk-ignore-all(flake8/F821): For SConstruct imports +# +# Firmware-specific Emscripten *link* settings for [env:native-wasm]. +# +# PlatformIO routes `build_flags` to the compile step, not the link step, so the +# WASM node's link-time emscripten settings have to be appended to LINKFLAGS +# here. The generic, app-agnostic flags (Asyncify, MODULARIZE, ALLOW_MEMORY_ +# GROWTH, the ES6 module shape) live in the platform-wasm builder; what belongs +# to *this firmware* is: +# +# * EXPORT_NAME — the ES-module factory name consumers import. +# * EXPORTED_RUNTIME_METHODS — ccall/cwrap/callMain + FS/IDBFS/NODEFS/PATH and +# the string helpers the JS host + bridge drive. +# * EXPORTED_FUNCTIONS — the C entry points (the wasm_* API is also kept via +# EMSCRIPTEN_KEEPALIVE in source; _malloc/_free are +# needed so the host can marshal protobuf buffers). +# * ASYNCIFY_IMPORTS — the WebUSB seam: these imported C functions suspend +# the stack (Asyncify) while a WebUSB transfer awaits. +# +# Only attached to the wasm env (see extra_scripts in [env:native-wasm]); a guard keeps +# it inert if it is ever pulled into another env. +# +Import("env") + +if env["PIOENV"] == "native-wasm": + env.Append( + LINKFLAGS=[ + "-sEXPORT_NAME=createMeshNode", + "-sEXPORTED_RUNTIME_METHODS=ccall,cwrap,callMain,FS,IDBFS,NODEFS,PATH,HEAPU8,UTF8ToString,stringToUTF8", + "-sEXPORTED_FUNCTIONS=_main,_wasm_setup,_wasm_loop_once,_wasm_fs_sync," + "_wasm_set_region,_wasm_api_to_radio,_wasm_api_from_radio," + "_wasm_api_available,_wasm_api_is_connected,_wasm_set_lora_module," + "_wasm_set_lora_usb_ids,_wasm_set_lora_usb_serial," + "_wasm_set_lora_dio_config,_wasm_set_lora_spi_speed,_wasm_set_lora_pin," + "_malloc,_free", + "-sASYNCIFY_IMPORTS=webusb_open,webusb_transceive,webusb_digital_write," + "webusb_digital_read,webusb_close", + ] + ) diff --git a/src/Power.cpp b/src/Power.cpp index 8104b53a6..31d98dba9 100644 --- a/src/Power.cpp +++ b/src/Power.cpp @@ -838,6 +838,14 @@ void Power::reboot() NVIC_SystemReset(); #elif defined(ARCH_RP2040) rp2040.reboot(); +#elif defined(ARCH_PORTDUINO_WASM) + // Browser/headless WASM node: no in-process restart. notifyReboot above + // already let modules persist; hand off to the host (reboot() -> + // location.reload() in a tab, or Module.onReboot() headless). Deliberately + // skip the ARCH_PORTDUINO SPI/Wire/Serial teardown below — it would kill the + // radio with no actual restart to follow, leaving a wedged node. Must come + // before the ARCH_PORTDUINO arm: the wasm build defines both macros. + ::reboot(); #elif defined(ARCH_PORTDUINO) deInitApiServer(); #ifdef __linux__ diff --git a/src/main.cpp b/src/main.cpp index 333db08cd..e76e071a1 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,4 +1,7 @@ #include "configuration.h" +#ifdef ARCH_PORTDUINO_WASM +#include +#endif #if !MESHTASTIC_EXCLUDE_GPS #include "GPS.h" #endif @@ -97,7 +100,9 @@ NRF54L15Bluetooth *nrf54l15Bluetooth = nullptr; #ifdef ARCH_PORTDUINO #include "linux/LinuxHardwareI2C.h" +#ifndef ARCH_PORTDUINO_WASM // raspi HTTP server (ulfius/zlib/openssl) excluded in the browser/wasm build #include "mesh/raspihttp/PiWebServer.h" +#endif #include "platform/portduino/PortduinoGlue.h" #include #include @@ -1110,10 +1115,12 @@ void setup() if (!rIf) RECORD_CRITICALERROR(meshtastic_CriticalErrorCode_NO_RADIO); else { +#ifndef ARCH_PORTDUINO_WASM // Log bit rate to debug output LOG_DEBUG("LoRA bitrate = %f bytes / sec", (float(meshtastic_Constants_DATA_PAYLOAD_LEN) / (float(rIf->getPacketTime(meshtastic_Constants_DATA_PAYLOAD_LEN)))) * 1000); +#endif router->addInterface(std::move(rIf)); } @@ -1400,7 +1407,16 @@ void loop() #ifdef DEBUG_LOOP_TIMING LOG_DEBUG("main loop delay: %d", delayMsec); #endif +#ifdef ARCH_PORTDUINO_WASM + // Single-threaded wasm: mainDelay's InterruptableDelay is a pthread + // cond/mutex semaphore that no other thread can ever give(), and + // emscripten's single-threaded pthread_cond_timedwait busy-spins. Suspend + // cooperatively via Asyncify instead, capping idle sleep so the per-tick + // IRQ poll latency stays bounded (RX/TX-done is detected by polling). + emscripten_sleep(delayMsec > 50 ? 50 : delayMsec); +#else mainDelay.delay(delayMsec); +#endif } } #endif diff --git a/src/mesh/HardwareRNG.cpp b/src/mesh/HardwareRNG.cpp index a34a9477c..ead66ac5f 100644 --- a/src/mesh/HardwareRNG.cpp +++ b/src/mesh/HardwareRNG.cpp @@ -128,6 +128,9 @@ bool fill(uint8_t *buffer, size_t length, bool useRadioEntropy) if (generated == static_cast(length)) { filled = true; } +#elif defined(__EMSCRIPTEN__) + // Browser/wasm: no getrandom/arc4random — fall through to std::random_device, + // which emscripten backs with crypto.getRandomValues(). #else // arc4random_buf is available on Darwin/BSD and cannot fail. ::arc4random_buf(buffer, length); diff --git a/src/mesh/InterfacesTemplates.cpp b/src/mesh/InterfacesTemplates.cpp index 6f6b2d2a1..6e27b56f3 100644 --- a/src/mesh/InterfacesTemplates.cpp +++ b/src/mesh/InterfacesTemplates.cpp @@ -8,8 +8,10 @@ #include "SX126xInterface.h" #include "SX128xInterface.cpp" #include "SX128xInterface.h" +#ifndef ARCH_PORTDUINO_WASM // TCP socket API server excluded in the browser/wasm build #include "api/ServerAPI.cpp" #include "api/ServerAPI.h" +#endif // We need this declaration for proper linking in derived classes #if RADIOLIB_EXCLUDE_SX126X != 1 diff --git a/src/mesh/LR11x0Interface.cpp b/src/mesh/LR11x0Interface.cpp index 9aefb7a8a..3f3dc0a23 100644 --- a/src/mesh/LR11x0Interface.cpp +++ b/src/mesh/LR11x0Interface.cpp @@ -371,7 +371,11 @@ template bool LR11x0Interface::sleep() template int16_t LR11x0Interface::getCurrentRSSI() { +#ifdef ARCH_PORTDUINO_WASM + float rssi = lora.getRSSI(); // installed RadioLib's LR11x0 getRSSI() is 0-arg +#else float rssi = lora.getRSSI(false, true); +#endif return (int16_t)round(rssi); } #endif diff --git a/src/mesh/SX126xInterface.cpp b/src/mesh/SX126xInterface.cpp index 920df6000..a2e6e4d79 100644 --- a/src/mesh/SX126xInterface.cpp +++ b/src/mesh/SX126xInterface.cpp @@ -328,10 +328,18 @@ template void SX126xInterface::startReceive() setTransmitEnable(false); setStandby(); +#ifdef ARCH_PORTDUINO_WASM + // Continuous RX in the browser: duty-cycle sleep parks BUSY high between RX + // windows and stalls the slow WebUSB SPI link. No battery to save here. + int err = lora.startReceive(RADIOLIB_SX126X_RX_TIMEOUT_INF, MESHTASTIC_RADIOLIB_IRQ_RX_FLAGS); + const char *rxMethod = "startReceive"; +#else // We use a 16 bit preamble so this should save some power by letting radio sit in standby mostly. int err = lora.startReceiveDutyCycleAuto(preambleLength, 8, MESHTASTIC_RADIOLIB_IRQ_RX_FLAGS); + const char *rxMethod = "startReceiveDutyCycleAuto"; +#endif if (err != RADIOLIB_ERR_NONE) - LOG_ERROR("SX126X startReceiveDutyCycleAuto %s%d", radioLibErr, err); + LOG_ERROR("SX126X %s %s%d", rxMethod, radioLibErr, err); #ifdef ARCH_PORTDUINO if (err != RADIOLIB_ERR_NONE) portduino_status.LoRa_in_error = true; diff --git a/src/platform/portduino/PortduinoGlue.cpp b/src/platform/portduino/PortduinoGlue.cpp index 2c72d83cb..301223a2f 100644 --- a/src/platform/portduino/PortduinoGlue.cpp +++ b/src/platform/portduino/PortduinoGlue.cpp @@ -232,6 +232,18 @@ void portduinoSetup() concurrency::hasBeenSetup = true; consoleInit(); +#ifdef ARCH_PORTDUINO_WASM + // Browser build: no YAML/filesystem config. Apply a hardcoded SX1262/CH341 + // setup and create the WebUSB-backed Ch341Hal, then skip the Linux config path. + { + extern void wasm_config_apply(); + wasm_config_apply(); + ch341Hal = + new Ch341Hal(0, portduino_config.lora_usb_serial_num, portduino_config.lora_usb_vid, portduino_config.lora_usb_pid); + } + return; +#endif + if (portduino_config.force_simradio == true) { portduino_config.lora_module = use_simradio; } else if (configPath != nullptr) { @@ -275,10 +287,12 @@ void portduinoSetup() } } +#ifndef ARCH_PORTDUINO_WASM if (yamlOnly) { std::cout << portduino_config.emit_yaml() << std::endl; exit(EXIT_SUCCESS); } +#endif if (portduino_config.force_simradio) { std::cout << "Running in simulated mode." << std::endl; @@ -733,6 +747,16 @@ int initGPIOPin(int pinNum, const std::string &gpioChipName, int line) #endif } +#ifdef ARCH_PORTDUINO_WASM +// Browser node: configuration comes from the wasm_set_lora_* setters, not a YAML +// file. Reached only as dead code after portduinoSetup()'s early return; kept +// defined (and yaml-free) so those references still link. +bool loadConfig(const char *configPath) +{ + (void)configPath; + return false; +} +#else bool loadConfig(const char *configPath) { YAML::Node yamlConfig; @@ -1090,6 +1114,7 @@ bool loadConfig(const char *configPath) } return true; } +#endif // !ARCH_PORTDUINO_WASM // https://stackoverflow.com/questions/874134/find-out-if-string-ends-with-another-string-in-c static bool ends_with(std::string_view str, std::string_view suffix) @@ -1129,6 +1154,10 @@ bool MAC_from_string(std::string mac_str, uint8_t *dmac) std::string exec(const char *cmd) { // https://stackoverflow.com/a/478960 +#ifdef ARCH_PORTDUINO_WASM + (void)cmd; // no shell/popen in the browser — shell-outs degrade to empty + return ""; +#endif std::array buffer; std::string result; std::unique_ptr pipe(popen(cmd, "r"), pclose); @@ -1141,6 +1170,7 @@ std::string exec(const char *cmd) return result; } +#ifndef ARCH_PORTDUINO_WASM void readGPIOFromYaml(YAML::Node sourceNode, pinMapping &destPin, int pinDefault) { if (sourceNode.IsMap()) { @@ -1155,3 +1185,4 @@ void readGPIOFromYaml(YAML::Node sourceNode, pinMapping &destPin, int pinDefault destPin.gpiochip = portduino_config.lora_default_gpiochip; } } +#endif // !ARCH_PORTDUINO_WASM diff --git a/src/platform/portduino/PortduinoGlue.h b/src/platform/portduino/PortduinoGlue.h index b38cfca25..d13efd983 100644 --- a/src/platform/portduino/PortduinoGlue.h +++ b/src/platform/portduino/PortduinoGlue.h @@ -8,7 +8,12 @@ #include "LR11x0Interface.h" #include "Module.h" #include "mesh/generated/meshtastic/mesh.pb.h" +#ifndef ARCH_PORTDUINO_WASM +// The browser (WASM) node configures the radio via the wasm_set_lora_* setters +// instead of a YAML file, so it has no yaml-cpp dependency. Everything that uses +// YAML below (emit_yaml / loadConfig / readGPIOFromYaml) is likewise guarded. #include "yaml-cpp/yaml.h" +#endif extern struct portduino_status_struct { bool LoRa_in_error = false; @@ -64,7 +69,9 @@ bool loadConfig(const char *configPath); static bool ends_with(std::string_view str, std::string_view suffix); void getMacAddr(uint8_t *dmac); bool MAC_from_string(std::string mac_str, uint8_t *dmac); +#ifndef ARCH_PORTDUINO_WASM void readGPIOFromYaml(YAML::Node sourceNode, pinMapping &destPin, int pinDefault = RADIOLIB_NC); +#endif std::string exec(const char *cmd); extern struct portduino_config_struct { @@ -221,6 +228,7 @@ extern struct portduino_config_struct { &tbRightPin, &tbPressPin}; +#ifndef ARCH_PORTDUINO_WASM std::string emit_yaml() { YAML::Emitter out; @@ -569,4 +577,5 @@ extern struct portduino_config_struct { out << YAML::EndMap; // General return out.c_str(); } +#endif // !ARCH_PORTDUINO_WASM } portduino_config; diff --git a/src/platform/portduino/wasm/README.md b/src/platform/portduino/wasm/README.md new file mode 100644 index 000000000..c7549c676 --- /dev/null +++ b/src/platform/portduino/wasm/README.md @@ -0,0 +1,82 @@ +# ARCH_PORTDUINO_WASM — meshtasticd in WebAssembly (LoRa over WebUSB) + +Builds the full portduino firmware (`setup()`/`loop()`) to WebAssembly with +Emscripten, so a real Meshtastic node runs in a browser tab (or headless Node) +and drives a LoRa radio over **WebUSB** through a CH341 USB-to-SPI bridge — the +same `Ch341Hal` path the desktop `meshtasticd` uses, with the libusb backend +swapped for a WebUSB one. The desktop/native portduino build is untouched. + +## Layout (this dir is excluded from the native PlatformIO `build_src_filter`) + +| file | role | +| -------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `portduino_glue_wasm.cpp` | LoRa config (MeshToad default + `wasm_set_lora_*` setters, no YAML), VFS mount, region/MAC helpers, and the `wasm_api_*` PhoneAPI bridge | +| `portduino_main_wasm.cpp` | `wasm_setup()` / `wasm_loop_once()` — JS drives the cooperative loop | +| `libpinedio_webusb.c` | WebUSB libpinedio backend (sync C ↔ async WebUSB via Asyncify `EM_ASYNC_JS`) | +| `include/libpinedio-usb.h` | the 12-fn libpinedio API the backend implements | +| `stubs/` | `argp.h` shim + jsoncpp serializer stub (MQTT-only, excluded) | +| `js/` | the WebUSB runtime: `bridge.js` (implements the C backend's imports), `ch341.js` (CH341 transport), `protocol.js` (framing) | + +In-tree, six firmware sources carry small `#ifdef ARCH_PORTDUINO_WASM` guards +(single-threaded cooperative sleep, continuous RX, region default, RNG, etc.): +`src/main.cpp`, `src/mesh/{NodeDB,SX126xInterface,InterfacesTemplates,LR11x0Interface,HardwareRNG}.cpp`, +`src/platform/portduino/PortduinoGlue.cpp`. None affect non-wasm builds. + +## Build + +This is a normal PlatformIO env (`[env:native-wasm]`) built with the +[meshtastic/platform-wasm](https://github.com/meshtastic/platform-wasm) platform +(emcc/em++), exactly like any other board target. + +Prereq: an **Emscripten SDK** on `PATH` — `source /emsdk_env.sh` (or +`export EMSDK=`) so the platform builder can locate `emcc`. + +```sh +pio run -e native-wasm # emcc compile + Asyncify link +pio run -e native-wasm -t clean # wipe the build dir +``` + +Output: `.pio/build/native-wasm/meshnode.mjs` + `meshnode.wasm` (ES module, Asyncify, +factory `createMeshNode`, exports `_wasm_setup`, `_wasm_loop_once`, +`_wasm_fs_sync`, `_wasm_set_region`, `_wasm_api_to_radio`, `_wasm_api_from_radio`, +`_wasm_api_available`, `_wasm_api_is_connected`, the `_wasm_set_lora_*` setters). + +## Run + +The C backend imports `webusb_*` functions; `js/bridge.js` implements them on top +of `js/ch341.js`. Minimal host flow: + +```js +import createMeshNode from "./meshnode.mjs"; +import { CH341 } from "./js/ch341.js"; +import { createCH341Bridge } from "./js/bridge.js"; + +const dev = (await CH341.request()).device; // WebUSB device picker (Chromium) +const Module = await createMeshNode({ noInitialRun: true }); +Module.ch341 = createCH341Bridge(Module, dev); // wire WebUSB before boot +await Module.ccall("wasm_setup", null, [], [], { async: true }); +const pump = async () => { + await Module.ccall("wasm_loop_once", "number", [], [], { async: true }); + setTimeout(pump, 5); +}; +pump(); +``` + +**API control:** feed a `ToRadio` protobuf with `wasm_api_to_radio(ptr,len)` and +drain `FromRadio` with `wasm_api_from_radio(out,max)` — the firmware's own +`PhoneAPI`, unframed. The official `@meshtastic/core` SDK drives it through a +~40-line in-process transport (see the `meshtasticd-wasm-node` repo, which hosts +the dev server, the SDK-UI page, the headless node-usb runner, and the TCP :4403 +bridge for the Python CLI). WebUSB is Chromium-only. + +**Reboot:** the firmware can't restart itself in wasm, so a reboot (admin/phone +command, factory reset, or the 60 s stuck-TX watchdog) hands off to the host. In +a browser it calls `location.reload()` — NodeDB state survives via IDBFS, so the +node comes back with the same identity. Headless, provide a `Module.onReboot` +callback to handle it (re-instantiate the module, `process.exit()` for a +supervisor to restart, etc.); without one it just logs and keeps running. + +```js +const Module = await createMeshNode({ noInitialRun: true }); +Module.onReboot = () => process.exit(0); // optional; headless restart policy +``` diff --git a/src/platform/portduino/wasm/include/libpinedio-usb.h b/src/platform/portduino/wasm/include/libpinedio-usb.h new file mode 100644 index 000000000..419fecb25 --- /dev/null +++ b/src/platform/portduino/wasm/include/libpinedio-usb.h @@ -0,0 +1,79 @@ +// WebUSB/wasm-compatible drop-in for libch341-spi-userspace's public header. +// +// Same API surface as the upstream libpinedio-usb.h that the firmware's +// Ch341Hal (src/platform/portduino/USBHal.h) compiles against, but WITHOUT the +// libusb / pthread dependencies — the implementation (libpinedio_webusb.c) +// forwards to a JS WebUSB bridge via Emscripten. The struct keeps only the +// fields Ch341Hal actually touches (serial_number, product_string, in_error, +// options[]); GPIO/CS state now lives on the JS side. +#ifndef PINEDIO_USB_WEBUSB_H +#define PINEDIO_USB_WEBUSB_H +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +enum pinedio_int_pin { + PINEDIO_PIN_D0, + PINEDIO_PIN_D1, + PINEDIO_PIN_D2, + PINEDIO_PIN_D3, + PINEDIO_PIN_D4, + PINEDIO_PIN_D5, + PINEDIO_PIN_D6, + PINEDIO_PIN_D7, + PINEDIO_PIN_ERR, + PINEDIO_PIN_PEMP, + PINEDIO_PIN_INT, + PINEDIO_INT_PIN_MAX +}; + +enum pinedio_int_mode { + PINEDIO_INT_MODE_RISING = 0x01, + PINEDIO_INT_MODE_FALLING = 0x02, +}; + +enum pinedio_option { + PINEDIO_OPTION_AUTO_CS, + PINEDIO_OPTION_SEARCH_SERIAL, + PINEDIO_OPTION_VID, + PINEDIO_OPTION_PID, + PINEDIO_OPTION_MAX +}; + +struct pinedio_inst_int { + uint8_t previous_state; + enum pinedio_int_mode mode; + void (*callback)(void); +}; + +struct pinedio_inst { + bool in_error; + struct pinedio_inst_int interrupts[PINEDIO_INT_PIN_MAX]; + uint32_t options[PINEDIO_OPTION_MAX]; + char serial_number[9]; + char product_string[97]; +}; + +typedef struct pinedio_inst pinedio_inst; + +int32_t pinedio_init(struct pinedio_inst *inst, void *driver); +int32_t pinedio_set_option(struct pinedio_inst *inst, enum pinedio_option option, uint32_t value); +int32_t pinedio_set_pin_mode(struct pinedio_inst *inst, uint32_t pin, uint32_t mode); +int32_t pinedio_digital_write(struct pinedio_inst *inst, uint32_t pin, bool active); +int32_t pinedio_set_cs(struct pinedio_inst *inst, bool active); +int32_t pinedio_write_read(struct pinedio_inst *inst, uint8_t *writearr, uint32_t writecnt, uint8_t *readarr, uint32_t readcnt); +int32_t pinedio_transceive(struct pinedio_inst *inst, uint8_t *write_buf, uint8_t *read_buf, uint32_t count); +int32_t pinedio_digital_read(struct pinedio_inst *inst, uint32_t pin); +int32_t pinedio_get_irq_state(struct pinedio_inst *inst, uint32_t pin); +int32_t pinedio_attach_interrupt(struct pinedio_inst *inst, enum pinedio_int_pin int_pin, enum pinedio_int_mode int_mode, + void (*callback)(void)); +int32_t pinedio_deattach_interrupt(struct pinedio_inst *inst, enum pinedio_int_pin int_pin); +void pinedio_deinit(struct pinedio_inst *inst); + +#ifdef __cplusplus +} +#endif +#endif // PINEDIO_USB_WEBUSB_H diff --git a/src/platform/portduino/wasm/js/bridge.js b/src/platform/portduino/wasm/js/bridge.js new file mode 100644 index 000000000..cfbdf9332 --- /dev/null +++ b/src/platform/portduino/wasm/js/bridge.js @@ -0,0 +1,115 @@ +// JS side of the wasm WebUSB bridge. The C backend (libpinedio_webusb.c) calls +// Module.ch341. from EM_ASYNC_JS/EM_JS. This wraps the same CH341 +// transport used by the pure-JS probe, marshalling buffers in/out of the wasm +// heap. +// +// Wiring (with MODULARIZE'd Emscripten output): +// const device = (await CH341.request()).device; // user gesture +// const Module = await createModule({ noInitialRun: true }); +// Module.ch341 = createCH341Bridge(Module, device); +// Module.callMain([]); // runs C main() +// +// IMPORTANT: a wasm heap can grow across an `await`, so always re-read +// Module.HEAPU8 *after* awaiting, never cache it across a suspension point. + +import { CH341 } from "./ch341.js"; + +export function createCH341Bridge(Module, device) { + let ch = null; + + function writeCString(str, ptr, max) { + const bytes = new TextEncoder().encode(str); + const n = Math.min(bytes.length, max - 1); + const heap = Module.HEAPU8; + heap.set(bytes.subarray(0, n), ptr); + heap[ptr + n] = 0; + } + + return { + // vid/pid/serial come from the C side; if a device was already handed in + // (typical — selected via a user gesture), just open that one. + async open(vid, pid, serial) { + let dev = device; + if (!dev) { + dev = await CH341.tryReconnect({ vid, pid }).then((c) => + c ? c.device : null, + ); + if (!dev) return -2; // no granted device; page must requestDevice first + } + // First-connect is flaky: the WebUSB interface can be momentarily + // unclaimable right after the grant, or still held by a prior session + // (yields the transient "Could not open SPI: -1"). Retry with a short + // backoff, resetting the device between attempts so claimInterface doesn't + // trip over a half-open device. + const attempts = 4; + for (let i = 0; i < attempts; i++) { + ch = new CH341(dev); + try { + await ch.open(); + return 0; + } catch (e) { + console.warn(`CH341 open attempt ${i + 1}/${attempts} failed:`, e); + try { + await dev.close(); // release/close so the next attempt starts clean + } catch (_) {} + if (i < attempts - 1) + await new Promise((r) => setTimeout(r, 200 * (i + 1))); + } + } + console.error("CH341 open failed after", attempts, "attempts"); + return -1; + }, + + // Read `count` bytes from the heap at writePtr, full-duplex transfer, write + // the result back at readPtr. Returns 0 / negative. + async transceive(writePtr, readPtr, count) { + try { + const out = Module.HEAPU8.slice(writePtr, writePtr + count); // copy before await + const inBuf = await ch.transceive(out); + Module.HEAPU8.set(inBuf, readPtr); // re-read HEAPU8 (may have grown) + return 0; + } catch (e) { + console.error("transceive failed:", e); + return -1; + } + }, + + async digitalWrite(pin, value) { + try { + await ch.digitalWrite(pin, value); + return 0; + } catch (e) { + return -1; + } + }, + + async digitalRead(pin) { + try { + return await ch.digitalRead(pin); + } catch (e) { + return -1; + } + }, + + setPinMode(pin, output) { + ch.setPinMode(pin, output); + }, + + setAutoCS(enabled) { + if (ch) ch.autoCS = enabled; + }, + + getSerial(ptr, max) { + writeCString(ch?.serial || "", ptr, max); + }, + + getProduct(ptr, max) { + writeCString(ch?.product || "", ptr, max); + }, + + async close() { + if (ch) await ch.close(); + ch = null; + }, + }; +} diff --git a/src/platform/portduino/wasm/js/ch341.js b/src/platform/portduino/wasm/js/ch341.js new file mode 100644 index 000000000..f4da3843f --- /dev/null +++ b/src/platform/portduino/wasm/js/ch341.js @@ -0,0 +1,186 @@ +// WebUSB driver for the CH341 USB-to-SPI bridge. +// +// This is the browser-side transport. It mirrors the public surface that the +// firmware's Ch341Hal (src/platform/portduino/USBHal.h) needs from the +// libpinedio C API — transceive / digitalWrite / digitalRead / setPinMode / +// setCS — but backed by async WebUSB instead of libusb. +// +// Every method is async (WebUSB is Promise-only). The wasm build will call the +// same WebUSB primitives from C via Asyncify so a synchronous C SPI transfer +// can await these. Here, used directly, it's the pure-JS transport for the +// hardware checkpoint (web/probe.js). + +import * as proto from "./protocol.js"; + +// Opt-in per-op USB tracing (node: DEBUG_USB=1). When a run wedges, the tail of +// the log shows the last USB ops + timing, so a stuck BUSY poll vs a stalled +// transfer is obvious. +let __usbOp = 0; +const __now = () => + typeof performance !== "undefined" ? performance.now() : Date.now(); +const __dbg = + typeof process !== "undefined" && process?.env?.DEBUG_USB + ? (m) => console.error(`[usb#${++__usbOp} ${__now().toFixed(0)}ms] ${m}`) + : null; +const __hex = (a, n = 6) => + Array.from(a.slice(0, n)) + .map((b) => b.toString(16).padStart(2, "0")) + .join(""); + +export class CH341 { + /** @param {USBDevice} device */ + constructor(device) { + this.device = device; + this.dMode = 0; // D0..D7 direction bitfield (1 = output) + this.dState = 0; // D0..D7 output level bitfield + this.autoCS = true; // toggle CS around each transceive + this.csPin = 0; // D0 is CS on the common CH341 LoRa adapters + this.csActiveLow = true; // NSS is active-low: assert (select) = drive CS LOW + this.interfaceNumber = 0; + } + + // Prompt the user to pick a CH341 (must be called from a user gesture). + static async request({ vid = proto.VID, pid = proto.PID } = {}) { + if (!("usb" in navigator)) + throw new Error("WebUSB unavailable (use Chromium over https/localhost)"); + const device = await navigator.usb.requestDevice({ + filters: [{ vendorId: vid, productId: pid }], + }); + return new CH341(device); + } + + // Reconnect to an already-granted device without a prompt, if present. + static async tryReconnect({ vid = proto.VID, pid = proto.PID } = {}) { + if (!("usb" in navigator)) return null; + const devices = await navigator.usb.getDevices(); + const device = devices.find( + (d) => d.vendorId === vid && d.productId === pid, + ); + return device ? new CH341(device) : null; + } + + async open() { + await this.device.open(); + // Don't read .configuration (the node-usb backend throws "device is not + // configured" instead of returning null); just (re)select config 1, which is + // idempotent in the browser if it's already active. + try { + await this.device.selectConfiguration(1); + } catch (_) {} + // The SPI bridge lives on interface 0 with bulk EP 0x02/0x82. + await this.device.claimInterface(this.interfaceNumber); + this.serial = this.device.serialNumber || ""; + this.product = this.device.productName || ""; + // CH341A SPI bus pins MUST be configured as outputs or the chip never + // clocks SCK/MOSI and every MISO byte reads back 0xFF. D3=SCK, D5=MOSI + // (D7=MISO stays input). This mirrors Ch341Hal's constructor + // (USBHal.h: pinedio_set_pin_mode(&pinedio, 3, true) / (5, true)). + this.setPinMode(3, true); // SCK (DCK) + this.setPinMode(5, true); // MOSI (DOUT) + this.setPinMode(this.csPin, true); // CS (NSS) + await this.setCS(false); // transmits direction bits + idles CS deasserted + } + + async close() { + try { + await this.device.releaseInterface(this.interfaceNumber); + } catch (_) {} + try { + await this.device.close(); + } catch (_) {} + } + + async _out(bytes) { + const r = await this.device.transferOut(proto.WRITE_EP, bytes); + if (r.status !== "ok") throw new Error(`USB OUT ${r.status}`); + return r.bytesWritten; + } + + async _in(len) { + const r = await this.device.transferIn(proto.READ_EP, len); + if (r.status !== "ok") throw new Error(`USB IN ${r.status}`); + return new Uint8Array(r.data.buffer, r.data.byteOffset, r.data.byteLength); + } + + // GPIO direction is recorded here and applied on the next digitalWrite, + // exactly as libpinedio does (its set_pin_mode does not transmit on its own). + setPinMode(pin, output) { + if (output) this.dMode |= 1 << pin; + else this.dMode &= ~(1 << pin); + } + + async digitalWrite(pin, value) { + if (value) this.dState |= 1 << pin; + else this.dState &= ~(1 << pin); + await this._out(proto.buildUioOut(this.dState, this.dMode)); + } + + async setCS(active) { + // active === "chip selected". On active-low NSS (the usual case) that means + // driving the CS line LOW. libpinedio's upstream AUTO_CS drove it HIGH, + // which leaves the radio deselected on these adapters (MISO reads 0xFF). + const level = this.csActiveLow ? (active ? 0 : 1) : active ? 1 : 0; + return this.digitalWrite(this.csPin, level); + } + + async digitalRead(pin) { + __dbg && __dbg(`dR D${pin} start`); + await this._out(proto.buildGetInput()); + const reply = await this._in(6); + const v = proto.inputPin(reply, pin); + __dbg && __dbg(`dR D${pin}=${v} (in ${reply.length}B)`); + return v; + } + + // Full-duplex SPI transfer: returns a Uint8Array of the same length as the + // write buffer (MISO sampled for every byte clocked out). + async transceive(writeBytes) { + const data = + writeBytes instanceof Uint8Array + ? writeBytes + : Uint8Array.from(writeBytes); + __dbg && __dbg(`tx ${data.length}B out=${__hex(data)} start`); + if (this.autoCS) await this.setCS(true); + try { + const packets = proto.buildSpiStreamPackets(data); + const read = new Uint8Array(data.length); + let off = 0; + for (const pkt of packets) { + const dataLen = pkt.length - 1; + await this._out(pkt); + // WebUSB transferIn can return FEWER bytes than requested; accumulate + // until we have all dataLen MISO bytes, or the device returns nothing. + // Treating a short read's missing bytes as 0 corrupts the SPI response + // and puts the radio in a bad state (intermittent hangs during init). + let got = 0; + while (got < dataLen) { + const r = await this._in(dataLen - got); + // A zero-length read means the device gave us nothing while MISO bytes + // were still outstanding. Leaving them 0 would silently corrupt the SPI + // response, so fail loudly instead of returning a partial buffer. + if (r.length === 0) + throw new Error( + `CH341 SPI short read: ${got}/${dataLen} MISO bytes (device returned 0)`, + ); + for (let i = 0; i < r.length; i++) + read[off + got + i] = proto.reverseByte(r[i]); + got += r.length; + } + off += dataLen; + } + __dbg && __dbg(`tx ${data.length}B in=${__hex(read)} done`); + return read; + } finally { + if (this.autoCS) await this.setCS(false); + } + } + + // Convenience: write-then-read with CS held across the whole exchange (used + // for register reads where the read phase follows command+address bytes). + async writeRead(writeBytes, readLen) { + const tx = new Uint8Array(writeBytes.length + readLen); + tx.set(writeBytes, 0); // read phase clocks 0x00 + const rx = await this.transceive(tx); + return rx.slice(writeBytes.length); + } +} diff --git a/src/platform/portduino/wasm/js/protocol.js b/src/platform/portduino/wasm/js/protocol.js new file mode 100644 index 000000000..01f1e7396 --- /dev/null +++ b/src/platform/portduino/wasm/js/protocol.js @@ -0,0 +1,96 @@ +// CH341 USB-to-SPI wire protocol — pure framing functions, no WebUSB/DOM. +// +// Faithful port of the framing in libch341-spi-userspace (libpinedio-usb.c): +// - SPI is streamed with command 0xA8; each USB packet is <=32 bytes +// (1 command byte + up to 31 data bytes). The CH341 is bit-reversed on the +// wire, so every SPI byte (TX and RX) is bit-reversed. +// - GPIO (D0..D7) is driven with UIO stream command 0xAB. D0 is wired to CS. +// - Inputs are read with the 0xA0 status command. +// +// These functions are deliberately side-effect free so they can be unit-tested +// under node without any hardware, and reused verbatim by both the browser +// driver (src/ch341.js) and the wasm C backend's JS glue. + +export const VID = 0x1a86; +export const PID = 0x5512; + +// WebUSB endpoint *numbers* (direction is implied by transferIn/transferOut). +// libpinedio uses EP 0x02 (OUT) and 0x82 (IN) — both endpoint number 2. +export const WRITE_EP = 2; +export const READ_EP = 2; + +export const CMD_SPI_STREAM = 0xa8; +export const CMD_UIO_STREAM = 0xab; +export const UIO_STM_OUT = 0x80; +export const UIO_STM_DIR = 0x40; +export const UIO_STM_END = 0x20; +export const CMD_GET_STATUS = 0xa0; + +export const PACKET_LENGTH = 0x20; // 32 +export const DATA_PER_PACKET = PACKET_LENGTH - 1; // 31 + +// Reverse the bit order of a byte (CH341 clocks SPI MSB/LSB swapped). +export function reverseByte(x) { + x &= 0xff; + x = ((x >> 1) & 0x55) | ((x << 1) & 0xaa); + x = ((x >> 2) & 0x33) | ((x << 2) & 0xcc); + x = ((x >> 4) & 0x0f) | ((x << 4) & 0xf0); + return x & 0xff; +} + +// Split an SPI write buffer into bit-reversed 0xA8 stream packets. +// Returns an array of Uint8Array, each [0xA8, rev(b0), rev(b1), ...] with at +// most 31 data bytes. The number of SPI data bytes equals writeBytes.length +// (1:1 full-duplex), so the expected total read length is writeBytes.length. +export function buildSpiStreamPackets(writeBytes) { + const data = + writeBytes instanceof Uint8Array ? writeBytes : Uint8Array.from(writeBytes); + const packets = []; + for (let off = 0; off < data.length; off += DATA_PER_PACKET) { + const n = Math.min(DATA_PER_PACKET, data.length - off); + const pkt = new Uint8Array(1 + n); + pkt[0] = CMD_SPI_STREAM; + for (let i = 0; i < n; i++) pkt[1 + i] = reverseByte(data[off + i]); + packets.push(pkt); + } + return packets; +} + +// Un-reverse a buffer of bytes read back from the device (in place on a copy). +export function unreverseBytes(bytes) { + const out = new Uint8Array(bytes.length); + for (let i = 0; i < bytes.length; i++) out[i] = reverseByte(bytes[i]); + return out; +} + +// UIO packet that drives output levels (D0..D5) and sets direction. +// Mirrors pinedio_digital_write: [0xAB, 0x80|state, 0x40|dir, 0x20]. +// state/dir are masked to 6 bits so they don't collide with the command bits. +export function buildUioOut(stateBits, modeBits) { + return new Uint8Array([ + CMD_UIO_STREAM, + UIO_STM_OUT | (stateBits & 0x3f), + UIO_STM_DIR | (modeBits & 0x3f), + UIO_STM_END, + ]); +} + +// UIO packet that sets direction only. Mirrors pinedio_set_pin_mode: +// [0xAB, 0x40|dir, 0x20]. +export function buildUioDir(modeBits) { + return new Uint8Array([ + CMD_UIO_STREAM, + UIO_STM_DIR | (modeBits & 0x3f), + UIO_STM_END, + ]); +} + +// Status/input request. Device replies with up to 6 bytes; D0..D7 live in [0]. +export function buildGetInput() { + return new Uint8Array([CMD_GET_STATUS]); +} + +// Read a single D0..D7 input level from a status reply (byte 0 holds D0..D7). +export function inputPin(statusReply, pin) { + return (statusReply[0] >> pin) & 1; +} diff --git a/src/platform/portduino/wasm/libpinedio_webusb.c b/src/platform/portduino/wasm/libpinedio_webusb.c new file mode 100644 index 000000000..c095e1828 --- /dev/null +++ b/src/platform/portduino/wasm/libpinedio_webusb.c @@ -0,0 +1,172 @@ +// libpinedio API implemented over WebUSB (Emscripten), replacing the libusb +// backend for the wasm build. Each SPI/GPIO operation forwards to a JS bridge +// (Module.ch341, see wasm/bridge.js) that owns the actual USBDevice and reuses +// the framing in src/protocol.js. +// +// KEY DESIGN POINTS +// * EM_ASYNC_JS makes a *synchronous-looking* C call await a WebUSB Promise. +// This requires linking with Asyncify (or JSPI). One async suspend per SPI +// transfer (not per packet) keeps the Asyncify cost down. +// * The "single outstanding USB op" invariant is automatic: every C call here +// awaits a complete JS operation before returning, and the node is +// single-threaded-cooperative, so no transfer overlaps another. +// * attachInterrupt does NOT spawn a thread (the upstream lib polls a pthread +// over USB). We record the callback only and rely on the firmware's +// pollMissedIrqs()/IRQ-flag polling from the main loop instead. +// * After any `await`, the wasm heap may have grown (ALLOW_MEMORY_GROWTH), so +// the JS bridge MUST re-read Module.HEAPU8 when writing results back. + +#include "libpinedio-usb.h" +#include +#include + +// ---- JS bridge imports ------------------------------------------------------ +// Async (suspend) operations: + +EM_ASYNC_JS(int, webusb_open, (int vid, int pid, int serialPtr), { + const serial = serialPtr ? UTF8ToString(serialPtr) : ""; + return await Module.ch341.open(vid, pid, serial); +}); + +EM_ASYNC_JS(int, webusb_transceive, (int writePtr, int readPtr, int count), + { return await Module.ch341.transceive(writePtr, readPtr, count); }); + +EM_ASYNC_JS(int, webusb_digital_write, (int pin, int value), { return await Module.ch341.digitalWrite(pin, value); }); + +EM_ASYNC_JS(int, webusb_digital_read, (int pin), { return await Module.ch341.digitalRead(pin); }); + +EM_ASYNC_JS(void, webusb_close, (void), { + if (Module.ch341) + await Module.ch341.close(); +}); + +// Synchronous (no USB) operations: + +EM_JS(void, webusb_set_pin_mode, (int pin, int output), { Module.ch341.setPinMode(pin, !!output); }); +EM_JS(void, webusb_set_auto_cs, (int enabled), { Module.ch341.setAutoCS(!!enabled); }); +EM_JS(void, webusb_get_serial, (int ptr, int max), { Module.ch341.getSerial(ptr, max); }); +EM_JS(void, webusb_get_product, (int ptr, int max), { Module.ch341.getProduct(ptr, max); }); + +// ---- libpinedio API --------------------------------------------------------- + +int32_t pinedio_set_option(struct pinedio_inst *inst, enum pinedio_option option, uint32_t value) +{ + if (option < PINEDIO_OPTION_MAX) + inst->options[option] = value; + if (option == PINEDIO_OPTION_AUTO_CS) + webusb_set_auto_cs((int)value); + return 0; +} + +int32_t pinedio_init(struct pinedio_inst *inst, void *driver) +{ + (void)driver; + inst->in_error = false; + for (int i = 0; i < PINEDIO_INT_PIN_MAX; i++) + inst->interrupts[i].callback = NULL; + + uint32_t vid = inst->options[PINEDIO_OPTION_VID] ? inst->options[PINEDIO_OPTION_VID] : 0x1A86; + uint32_t pid = inst->options[PINEDIO_OPTION_PID] ? inst->options[PINEDIO_OPTION_PID] : 0x5512; + int serialPtr = inst->options[PINEDIO_OPTION_SEARCH_SERIAL] ? (int)inst->serial_number : 0; + + int ret = webusb_open((int)vid, (int)pid, serialPtr); + if (ret != 0) { + inst->in_error = true; + return ret < 0 ? ret : -2; + } + // Honor the configured Auto-CS. Ch341Hal sets PINEDIO_OPTION_AUTO_CS=0 (CS is + // left to RadioLib's NSS, which drives it active-low correctly); it has not + // been set yet at init, so this defaults off until Ch341Hal applies it. + webusb_set_auto_cs(inst->options[PINEDIO_OPTION_AUTO_CS] ? 1 : 0); + webusb_get_serial((int)inst->serial_number, sizeof(inst->serial_number)); + webusb_get_product((int)inst->product_string, sizeof(inst->product_string)); + return 0; +} + +int32_t pinedio_set_pin_mode(struct pinedio_inst *inst, uint32_t pin, uint32_t mode) +{ + (void)inst; + webusb_set_pin_mode((int)pin, (int)mode); + return 0; +} + +int32_t pinedio_digital_write(struct pinedio_inst *inst, uint32_t pin, bool active) +{ + int ret = webusb_digital_write((int)pin, active ? 1 : 0); + if (ret < 0) + inst->in_error = true; + return ret; +} + +int32_t pinedio_set_cs(struct pinedio_inst *inst, bool active) +{ + return pinedio_digital_write(inst, 0, active); // D0 is CS +} + +int32_t pinedio_transceive(struct pinedio_inst *inst, uint8_t *write_buf, uint8_t *read_buf, uint32_t count) +{ + int ret = webusb_transceive((int)write_buf, (int)read_buf, (int)count); + if (ret < 0) { + inst->in_error = true; + return -1; + } + return 0; +} + +int32_t pinedio_write_read(struct pinedio_inst *inst, uint8_t *writearr, uint32_t writecnt, uint8_t *readarr, uint32_t readcnt) +{ + // Not on Ch341Hal's hot path; emulate with a single full-duplex transfer. + uint32_t total = writecnt + readcnt; + uint8_t buf[total]; // VLA; transfers here are small (register reads) + memcpy(buf, writearr, writecnt); + memset(buf + writecnt, 0, readcnt); + int ret = webusb_transceive((int)buf, (int)buf, (int)total); + if (ret < 0) { + inst->in_error = true; + return -1; + } + memcpy(readarr, buf + writecnt, readcnt); + return 0; +} + +int32_t pinedio_digital_read(struct pinedio_inst *inst, uint32_t pin) +{ + int ret = webusb_digital_read((int)pin); + if (ret < 0) { + inst->in_error = true; + return ret; + } + return ret; +} + +int32_t pinedio_get_irq_state(struct pinedio_inst *inst, uint32_t pin) +{ + return pinedio_digital_read(inst, pin); +} + +// No poll thread: record the callback so enable/disableInterrupt bookkeeping in +// RadioLib works; actual RX/TX detection is by polling IRQ flags in the loop. +int32_t pinedio_attach_interrupt(struct pinedio_inst *inst, enum pinedio_int_pin int_pin, enum pinedio_int_mode int_mode, + void (*callback)(void)) +{ + if (int_pin >= PINEDIO_INT_PIN_MAX) + return -1; + inst->interrupts[int_pin].previous_state = 255; + inst->interrupts[int_pin].mode = int_mode; + inst->interrupts[int_pin].callback = callback; + return 0; +} + +int32_t pinedio_deattach_interrupt(struct pinedio_inst *inst, enum pinedio_int_pin int_pin) +{ + if (int_pin >= PINEDIO_INT_PIN_MAX) + return -1; + inst->interrupts[int_pin].callback = NULL; + return 0; +} + +void pinedio_deinit(struct pinedio_inst *inst) +{ + webusb_close(); + inst->in_error = false; +} diff --git a/src/platform/portduino/wasm/portduino_glue_wasm.cpp b/src/platform/portduino/wasm/portduino_glue_wasm.cpp new file mode 100644 index 000000000..01b02e924 --- /dev/null +++ b/src/platform/portduino/wasm/portduino_glue_wasm.cpp @@ -0,0 +1,426 @@ +// WASM-side portduino glue (ARCH_PORTDUINO_WASM). Replaces the Linux YAML/ +// filesystem config path with a CH341 setup (a MeshToad by default, or whatever +// the JS host pre-sets via the wasm_set_lora_* setters), mounts a persistent FS +// (IDBFS in the browser / NODEFS headless), resolves a per-node MAC, and bridges +// the firmware's PhoneAPI to JS (wasm_api_*). Also supplies +// delay()/yield()->emscripten_sleep and the other framework symbols normally +// provided by the excluded linux/LinuxCommon.cpp. +// +// Wiring in the firmware tree (already in place, all #ifdef ARCH_PORTDUINO_WASM): +// - PortduinoGlue.cpp portduinoSetup(): calls wasm_config_apply() and +// constructs the WebUSB-backed Ch341Hal, then returns before the YAML path. +// - exec() short-circuits to "" (no popen/shell in the browser). +// Downstream is unchanged: Ch341Hal -> libpinedio_webusb.c -> WebUSB. + +#include "CryptoEngine.h" // crypto->ensurePkiKeys() +#include "MeshRadio.h" // initRegion() +#include "MeshService.h" // service->reloadConfig() +#include "NodeDB.h" // config, owner globals + SEGMENT_CONFIG +#include "PhoneAPI.h" // the transport-agnostic client API seam +#include "PortduinoFS.h" // portduinoVFS +#include "PortduinoGlue.h" // declares `portduino_config` + Ch341Hal +#include "RadioInterface.h" // RadioInterface::validateConfig*, instance +#include +#include +#include +#include +#include + +// Ask the JS host to persist the emscripten FS to its backing store. In the +// browser the /meshdata mount is IDBFS, so this flushes MEMFS->IndexedDB +// (async; fire-and-forget). Under headless node /meshdata is NODEFS (writes are +// already synchronous on the host fs) so syncfs is a harmless no-op there. +extern "C" EMSCRIPTEN_KEEPALIVE void wasm_fs_sync() +{ + // Coalesce: IDBFS syncfs is async, and overlapping syncs warn "2 FS.syncfs + // operations in flight". Never run two at once — if one is in flight, mark a + // pending re-sync and let the running one chain it when it finishes. + // NOTE: loose != / == and string ops only — clang-format mangles !== and + // /regex/ literals inside EM_ASM JS (splitting them into invalid tokens). + EM_ASM({ + try { + if (typeof FS == "undefined" || !FS.syncfs) + return; + if (Module.__fsSyncing) { + Module.__fsSyncPending = true; + return; + } + var run = function() + { + Module.__fsSyncing = true; + Module.__fsSyncPending = false; + FS.syncfs( + false, function(err) { + Module.__fsSyncing = false; + if (err) + console.warn("syncfs:", err); + if (Module.__fsSyncPending) + run(); + }); + }; + run(); + } catch (e) { + console.warn("syncfs threw:", e); + } + }); +} + +// Point the portduino VFS at /meshdata so NodeDB/config saves succeed (the +// framework main.cpp we replaced normally does this; without it every save fails +// with "File system is not mounted"). The JS host has already FS.mount'ed an +// IDBFS (browser) or NODEFS (headless) backend at /meshdata for real persistence +// (see web/fs-setup.js); here we just ensure the subtree exists and set the root. +void wasm_fs_mount() +{ + mkdir("/meshdata", 0777); + mkdir("/meshdata/prefs", 0777); + mkdir("/meshdata/oem", 0777); + portduinoVFS->mountpoint("/meshdata"); +} + +// Resolve a per-instance MAC address (12 uppercase hex chars, no colons — the +// MAC_from_string format getMacAddr() expects). The lower 4 bytes become the +// 32-bit NodeNum (pickNewNodeNum: mac[2..5]), so this is the node's identity on +// the mesh — every browser node MUST get a distinct one or they collide on the +// same NodeNum. Priority: +// 1. MESH_MAC env (headless determinism / parity tests, e.g. DEAD00C0FFEE), +// 2. a value persisted in the /meshdata tree (survives reload/restart), +// 3. a freshly generated locally-administered random MAC, which we persist. +// Runs inside wasm_config_apply() (called by portduinoSetup, before setup()'s +// pickNewNodeNum), and /meshdata is already JS-mounted + populated by then. +static std::string wasm_resolve_mac() +{ + // 1) explicit override (works in node via process.env; ignored in browser). + char env[32] = {0}; + EM_ASM( + { + try { + var m = (typeof process != "undefined" && process.env && process.env.MESH_MAC) || ""; + stringToUTF8(String(m).split(":").join(""), $0, 32); + } catch (e) { + } + }, + env); + if (strlen(env) >= 12) + return std::string(env).substr(0, 12); + + // 2) persisted identity (raw POSIX path, independent of portduinoVFS mountpoint). + if (FILE *f = fopen("/meshdata/oem/mac", "rb")) { + char buf[13] = {0}; + size_t n = fread(buf, 1, 12, f); + fclose(f); + if (n == 12) + return std::string(buf, 12); + } + + // 3) generate a locally-administered, unicast MAC and persist it. + unsigned char m[6] = {0}; + EM_ASM( + { + try { + var c = (typeof crypto != "undefined" && crypto.getRandomValues) ? crypto : null; + if (c) + c.getRandomValues(HEAPU8.subarray($0, $0 + 6)); + else + for (var i = 0; i < 6; i++) + HEAPU8[$0 + i] = (Math.random() * 256) | 0; + } catch (e) { + for (var j = 0; j < 6; j++) + HEAPU8[$0 + j] = (Math.random() * 256) | 0; + } + }, + m); + m[0] = (m[0] | 0x02) & 0xFE; // locally administered (bit1=1), unicast (bit0=0) + char hex[13]; + snprintf(hex, sizeof(hex), "%02X%02X%02X%02X%02X%02X", m[0], m[1], m[2], m[3], m[4], m[5]); + if (FILE *f = fopen("/meshdata/oem/mac", "wb")) { + fwrite(hex, 1, 12, f); + fclose(f); + wasm_fs_sync(); // browser: push the new identity to IndexedDB + } + return std::string(hex, 12); +} + +// ---- Per-adapter config setters -------------------------------------------- +// JS may call these BEFORE wasm_setup() to drive a non-MeshToad CH341 LoRa +// adapter. wasm_set_lora_module is the trigger: left unset (use_simradio, the +// struct default), wasm_config_apply() falls back to the MeshToad defaults. +extern "C" EMSCRIPTEN_KEEPALIVE void wasm_set_lora_module(int module_enum) +{ + portduino_config.lora_module = (lora_module_enum)module_enum; +} +extern "C" EMSCRIPTEN_KEEPALIVE void wasm_set_lora_usb_ids(int vid, int pid) +{ + portduino_config.lora_usb_vid = vid; + portduino_config.lora_usb_pid = pid; +} +extern "C" EMSCRIPTEN_KEEPALIVE void wasm_set_lora_usb_serial(const char *serial) +{ + portduino_config.lora_usb_serial_num = serial ? std::string(serial) : ""; +} +extern "C" EMSCRIPTEN_KEEPALIVE void wasm_set_lora_dio_config(int dio2_as_rf_switch, int dio3_tcxo_mv) +{ + portduino_config.dio2_as_rf_switch = (dio2_as_rf_switch != 0); + portduino_config.dio3_tcxo_voltage = dio3_tcxo_mv; +} +extern "C" EMSCRIPTEN_KEEPALIVE void wasm_set_lora_spi_speed(int hz) +{ + portduino_config.spiSpeed = hz; +} +// Pin name -> the matching portduino_config field. Sets .pin + .enabled to match +// the default path (the CH341 backend addresses by D-line number = .pin). +extern "C" EMSCRIPTEN_KEEPALIVE void wasm_set_lora_pin(const char *name, int pin) +{ + if (!name) + return; + std::string n(name); + pinMapping *t = nullptr; + if (n == "CS") + t = &portduino_config.lora_cs_pin; + else if (n == "IRQ") + t = &portduino_config.lora_irq_pin; + else if (n == "BUSY") + t = &portduino_config.lora_busy_pin; + else if (n == "RESET") + t = &portduino_config.lora_reset_pin; + else if (n == "RXEN") + t = &portduino_config.lora_rxen_pin; + else if (n == "TXEN") + t = &portduino_config.lora_txen_pin; + else if (n == "ANT_SW") + t = &portduino_config.lora_sx126x_ant_sw_pin; + if (t) { + t->pin = pin; + t->enabled = (pin != (int)RADIOLIB_NC); + } +} + +// LoRa adapter config. A MeshToad (E22/SX1262 over CH341) by default, or whatever +// the JS host pre-set via the wasm_set_lora_* setters before wasm_setup(). The +// browser/headless invariants (CH341 SPI dev, no screen/GPS, MAC) always apply. +// C++ linkage to match the `extern void wasm_config_apply();` decl in PortduinoGlue.cpp. +void wasm_config_apply() +{ + if (portduino_config.lora_module == use_simradio) { + // Nothing pre-set by JS -> MeshToad E22/SX1262 defaults. + portduino_config.lora_module = use_sx1262; + portduino_config.lora_usb_vid = 0x1A86; + portduino_config.lora_usb_pid = 0x5512; + portduino_config.lora_usb_serial_num = ""; // first matching device + portduino_config.dio2_as_rf_switch = true; // E22 uses DIO2 as the TX/RX switch + portduino_config.dio3_tcxo_voltage = 1800; // 1.8 V TCXO + portduino_config.spiSpeed = 2000000; + // MeshToad CH341 D-line pin map (bin/config.d/lora-usb-meshtoad-e22.yaml). + auto setPin = [](pinMapping &p, int n) { + p.pin = n; + p.enabled = true; + }; + setPin(portduino_config.lora_cs_pin, 0); // CS = D0 + setPin(portduino_config.lora_irq_pin, 6); // IRQ = D6 + setPin(portduino_config.lora_busy_pin, 4); // BUSY = D4 + setPin(portduino_config.lora_reset_pin, 2); // RESET = D2 + setPin(portduino_config.lora_rxen_pin, 1); // RXen = D1 + } + portduino_config.lora_spi_dev = "ch341"; // every adapter here is WebUSB/CH341 + portduino_config.displayPanel = no_screen; + portduino_config.has_gps = false; + portduino_config.MaxNodes = 80; // small DB for the browser + // Per-instance unique MAC (persisted in /meshdata, env-overridable). The lower + // 4 bytes become this node's NodeNum; also short-circuits getMacAddr()'s popen. + portduino_config.mac_address = wasm_resolve_mac(); +} + +// Set the LoRa region at runtime from the UI (browser