2.8: NodeDB shrink, decoupling, and restructuring (#10413)

* 2.8: NodeDB refactor to decouple satellite entries and decrease size * Regen * Refactor node mute handling to use dedicated functions for clarity and consistency * Develop ref * Fix NodeDB review follow-ups Agent-Logs-Url: https://github.com/meshtastic/firmware/sessions/6b1d6cf6-ed6b-43b6-95cb-8e141757664e Co-authored-by: thebentern <9000580+thebentern@users.noreply.github.com> * Address review validation nits Agent-Logs-Url: https://github.com/meshtastic/firmware/sessions/6b1d6cf6-ed6b-43b6-95cb-8e141757664e Co-authored-by: thebentern <9000580+thebentern@users.noreply.github.com> * Trunk * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Extract legacy NodeDatabase migration * Fix remaining NodeDB review issues Agent-Logs-Url: https://github.com/meshtastic/firmware/sessions/c76b9a5a-7244-4fbc-9ef0-98091d8caaea Co-authored-by: thebentern <9000580+thebentern@users.noreply.github.com> * Fixes * Trunk * Fix latest review compile follow-ups Agent-Logs-Url: https://github.com/meshtastic/firmware/sessions/5198da01-ec4c-4c16-8a09-68b8e6d5d410 Co-authored-by: thebentern <9000580+thebentern@users.noreply.github.com> * Fix cppcheck style warnings Agent-Logs-Url: https://github.com/meshtastic/firmware/sessions/e60287ba-4ece-46e0-83d8-a6d89664c0bb Co-authored-by: thebentern <9000580+thebentern@users.noreply.github.com> * Change pointer type for mesh node in set_favorite function * Change pointer types for mesh node references to const in multiple applets * Add NodeDB layout v25 documentation and migration guidelines * Remove tests for uninitialized PacketHistory state due to undefined behavior * Fix code block formatting in copilot instructions --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
2026-05-09 15:12:10 -05:00
parent 1bcabb893b
commit 94bb21ecc7
58 changed files with 2605 additions and 643 deletions
@@ -4,12 +4,16 @@
 #include "concurrency/Lock.h"
 #include "mesh-pb-constants.h"
 #include "meshtastic/portnums.pb.h"
+#include <cstdint>
 #include <deque>
 #include <iterator>
 #include <string>
 #include <unordered_map>
 #include <vector>

+// NodeNum stored as raw uint32_t below; including MeshTypes.h here breaks the
+// LOG_WARN include order via Arduino.h/MemoryPool.h.
+
 // Make sure that we never let our packets grow too large for one BLE packet
 #define MAX_TO_FROM_RADIO_SIZE 512

@@ -22,6 +26,9 @@

 #define SPECIAL_NONCE_ONLY_CONFIG 69420
 #define SPECIAL_NONCE_ONLY_NODES 69421 // ( ͡° ͜ʖ ͡°)
+// Gradient sync: phone sends one of these to opt into thin-header + replay.
+#define SPECIAL_NONCE_GRADIENT_SYNC 69422
+#define SPECIAL_NONCE_GRADIENT_ONLY_NODES 69423

 /**
 * Provides our protobuf based API which phone/PC clients can use to talk to our device
@@ -45,7 +52,13 @@ class PhoneAPI
        STATE_SEND_CONFIG,          // Replacement for the old Radioconfig
        STATE_SEND_MODULECONFIG,    // Send Module specific config
        STATE_SEND_OTHER_NODEINFOS, // states progress in this order as the device sends to to the client
-        STATE_SEND_FILEMANIFEST,    // Send file manifest
+        // Drain satellite DBs as synthetic POSITION_APP / TELEMETRY_APP /
+        // NODE_STATUS_APP packets when the phone opted into gradient sync.
+        STATE_REPLAY_POSITIONS,
+        STATE_REPLAY_TELEMETRY,
+        STATE_REPLAY_ENVIRONMENT,
+        STATE_REPLAY_STATUS,
+        STATE_SEND_FILEMANIFEST, // Send file manifest
        STATE_SEND_COMPLETE_ID,
        STATE_SEND_PACKETS // send packets or debug strings
    };
@@ -88,6 +101,20 @@ class PhoneAPI
    // Protect nodeInfoForPhone + nodeInfoQueue because NimBLE callbacks run in a separate FreeRTOS task.
    concurrency::Lock nodeInfoMutex;

+    // Synthetic-packet replay queue (paced via prefetch).
+    std::deque<meshtastic_MeshPacket> replayQueue;
+    static constexpr size_t kReplayPrefetchDepth = 4;
+    // NodeNum snapshots taken at each replay phase start so concurrent
+    // satellite-map mutations don't invalidate iteration.
+    std::vector<uint32_t> replayPositionOrder;
+    std::vector<uint32_t> replayTelemetryOrder;
+    std::vector<uint32_t> replayEnvironmentOrder;
+    std::vector<uint32_t> replayStatusOrder;
+    size_t replayPositionIndex = 0;
+    size_t replayTelemetryIndex = 0;
+    size_t replayEnvironmentIndex = 0;
+    size_t replayStatusIndex = 0;
+
    meshtastic_ToRadio toRadioScratch = {
        0}; // this is a static scratch object, any data must be copied elsewhere before returning

@@ -137,6 +164,10 @@ class PhoneAPI

    bool isConnected() { return state != STATE_SEND_NOTHING; }
    bool isSendingPackets() { return state == STATE_SEND_PACKETS; }
+    bool clientWantsGradientSync() const
+    {
+        return config_nonce == SPECIAL_NONCE_GRADIENT_SYNC || config_nonce == SPECIAL_NONCE_GRADIENT_ONLY_NODES;
+    }

  protected:
    /// Our fromradio packet while it is being assembled
@@ -185,6 +216,18 @@ class PhoneAPI
    void releaseQueueStatusPhonePacket();

    void prefetchNodeInfos();
+    void beginReplayPositions();
+    void prefetchReplayPositions();
+    void beginReplayTelemetry();
+    void prefetchReplayTelemetry();
+    void beginReplayEnvironment();
+    void prefetchReplayEnvironment();
+    void beginReplayStatus();
+    void prefetchReplayStatus();
+    meshtastic_MeshPacket makeReplayPositionPacket(uint32_t num, const meshtastic_PositionLite &pos);
+    meshtastic_MeshPacket makeReplayTelemetryPacket(uint32_t num, const meshtastic_DeviceMetrics &metrics);
+    meshtastic_MeshPacket makeReplayEnvironmentPacket(uint32_t num, const meshtastic_EnvironmentMetrics &env);
+    meshtastic_MeshPacket makeReplayStatusPacket(uint32_t num, const meshtastic_StatusMessage &status);

    void releaseMqttClientProxyPhonePacket();