kevin/meshtastic_firmware
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

Added NodeDB fixtures and refactored to use std maps for better memory efficiency (#10464)

Browse Source 
* Added NodeDB fixtures and refactored to use std maps for better efficiency

* Defer NodeDB save during xmodem transfer to prevent mid-transfer fsFormat
This commit is contained in:
Ben Meadors
2026-05-12 17:23:29 -05:00
committed by GitHub
co-authored by GitHub
parent f3ae02c425
commit eead467ce6
18 changed files with 5839 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mesh/NodeDB.cpp
+148 -55
View File
@@ -25,6 +25,7 @@
#include "mesh/generated/meshtastic/deviceonly_legacy.pb.h"
#include "meshUtils.h"
#include "modules/NeighborInfoModule.h"
#include "xmodem.h"
#include <ErriezCRC32.h>
#include <algorithm>
#include <pb_decode.h>
@@ -160,6 +161,25 @@ uint32_t get_st7789_id(uint8_t cs, uint8_t sck, uint8_t mosi, uint8_t dc, uint8_
#endif
// When armed by loadFromDisk, the decode callback writes satellite entries
// straight into these maps instead of the temp vectors. Nullptr = legacy
// push_back-to-vector path for backup/restore and other decoders.
namespace
{
#if !MESHTASTIC_EXCLUDE_POSITIONDB
std::map<NodeNum, meshtastic_PositionLite> *s_decodePositionsTarget = nullptr;
#endif
#if !MESHTASTIC_EXCLUDE_TELEMETRYDB
std::map<NodeNum, meshtastic_DeviceMetrics> *s_decodeTelemetryTarget = nullptr;
#endif
#if !MESHTASTIC_EXCLUDE_ENVIRONMENTDB
std::map<NodeNum, meshtastic_EnvironmentMetrics> *s_decodeEnvironmentTarget = nullptr;
#endif
#if !MESHTASTIC_EXCLUDE_STATUSDB
std::map<NodeNum, meshtastic_StatusMessage> *s_decodeStatusTarget = nullptr;
#endif
} // namespace
bool meshtastic_NodeDatabase_callback(pb_istream_t *istream, pb_ostream_t *ostream, const pb_field_t *field)
{
const auto *iter = reinterpret_cast<const pb_field_iter_t *>(field);
@@ -174,10 +194,10 @@ bool meshtastic_NodeDatabase_callback(pb_istream_t *istream, pb_ostream_t *ostre
return false;
}
}
if (istream) {
meshtastic_NodeInfoLite node;
if (istream && istream->bytes_left) {
meshtastic_NodeInfoLite node = meshtastic_NodeInfoLite_init_zero;
auto *vec = static_cast<std::vector<meshtastic_NodeInfoLite> *>(iter->pData);
if (istream->bytes_left && pb_decode(istream, meshtastic_NodeInfoLite_fields, &node))
if (pb_decode(istream, meshtastic_NodeInfoLite_fields, &node))
vec->push_back(node);
}
return true;
@@ -192,11 +212,19 @@ bool meshtastic_NodeDatabase_callback(pb_istream_t *istream, pb_ostream_t *ostre
return false;
}
}
if (istream) {
meshtastic_NodePositionEntry entry;
auto *vec = static_cast<std::vector<meshtastic_NodePositionEntry> *>(iter->pData);
if (istream->bytes_left && pb_decode(istream, meshtastic_NodePositionEntry_fields, &entry))
if (istream && istream->bytes_left) {
meshtastic_NodePositionEntry entry = meshtastic_NodePositionEntry_init_zero;
if (pb_decode(istream, meshtastic_NodePositionEntry_fields, &entry)) {
#if !MESHTASTIC_EXCLUDE_POSITIONDB
if (s_decodePositionsTarget) {
if (entry.has_position)
(*s_decodePositionsTarget)[entry.num] = entry.position;
return true;
}
#endif
auto *vec = static_cast<std::vector<meshtastic_NodePositionEntry> *>(iter->pData);
vec->push_back(entry);
}
}
return true;
}
@@ -210,11 +238,19 @@ bool meshtastic_NodeDatabase_callback(pb_istream_t *istream, pb_ostream_t *ostre
return false;
}
}
if (istream) {
meshtastic_NodeTelemetryEntry entry;
auto *vec = static_cast<std::vector<meshtastic_NodeTelemetryEntry> *>(iter->pData);
if (istream->bytes_left && pb_decode(istream, meshtastic_NodeTelemetryEntry_fields, &entry))
if (istream && istream->bytes_left) {
meshtastic_NodeTelemetryEntry entry = meshtastic_NodeTelemetryEntry_init_zero;
if (pb_decode(istream, meshtastic_NodeTelemetryEntry_fields, &entry)) {
#if !MESHTASTIC_EXCLUDE_TELEMETRYDB
if (s_decodeTelemetryTarget) {
if (entry.has_device_metrics)
(*s_decodeTelemetryTarget)[entry.num] = entry.device_metrics;
return true;
}
#endif
auto *vec = static_cast<std::vector<meshtastic_NodeTelemetryEntry> *>(iter->pData);
vec->push_back(entry);
}
}
return true;
}
@@ -228,11 +264,19 @@ bool meshtastic_NodeDatabase_callback(pb_istream_t *istream, pb_ostream_t *ostre
return false;
}
}
if (istream) {
meshtastic_NodeStatusEntry entry;
auto *vec = static_cast<std::vector<meshtastic_NodeStatusEntry> *>(iter->pData);
if (istream->bytes_left && pb_decode(istream, meshtastic_NodeStatusEntry_fields, &entry))
if (istream && istream->bytes_left) {
meshtastic_NodeStatusEntry entry = meshtastic_NodeStatusEntry_init_zero;
if (pb_decode(istream, meshtastic_NodeStatusEntry_fields, &entry)) {
#if !MESHTASTIC_EXCLUDE_STATUSDB
if (s_decodeStatusTarget) {
if (entry.has_status)
(*s_decodeStatusTarget)[entry.num] = entry.status;
return true;
}
#endif
auto *vec = static_cast<std::vector<meshtastic_NodeStatusEntry> *>(iter->pData);
vec->push_back(entry);
}
}
return true;
}
@@ -246,11 +290,19 @@ bool meshtastic_NodeDatabase_callback(pb_istream_t *istream, pb_ostream_t *ostre
return false;
}
}
if (istream) {
meshtastic_NodeEnvironmentEntry entry;
auto *vec = static_cast<std::vector<meshtastic_NodeEnvironmentEntry> *>(iter->pData);
if (istream->bytes_left && pb_decode(istream, meshtastic_NodeEnvironmentEntry_fields, &entry))
if (istream && istream->bytes_left) {
meshtastic_NodeEnvironmentEntry entry = meshtastic_NodeEnvironmentEntry_init_zero;
if (pb_decode(istream, meshtastic_NodeEnvironmentEntry_fields, &entry)) {
#if !MESHTASTIC_EXCLUDE_ENVIRONMENTDB
if (s_decodeEnvironmentTarget) {
if (entry.has_environment_metrics)
(*s_decodeEnvironmentTarget)[entry.num] = entry.environment_metrics;
return true;
}
#endif
auto *vec = static_cast<std::vector<meshtastic_NodeEnvironmentEntry> *>(iter->pData);
vec->push_back(entry);
}
}
return true;
}
@@ -259,6 +311,42 @@ bool meshtastic_NodeDatabase_callback(pb_istream_t *istream, pb_ostream_t *ostre
}
}
void NodeDB::armNodeDatabaseDecodeTargets()
{
#if !MESHTASTIC_EXCLUDE_POSITIONDB
nodePositions.clear();
s_decodePositionsTarget = &nodePositions;
#endif
#if !MESHTASTIC_EXCLUDE_TELEMETRYDB
nodeTelemetry.clear();
s_decodeTelemetryTarget = &nodeTelemetry;
#endif
#if !MESHTASTIC_EXCLUDE_ENVIRONMENTDB
nodeEnvironment.clear();
s_decodeEnvironmentTarget = &nodeEnvironment;
#endif
#if !MESHTASTIC_EXCLUDE_STATUSDB
nodeStatus.clear();
s_decodeStatusTarget = &nodeStatus;
#endif
}
void NodeDB::disarmNodeDatabaseDecodeTargets()
{
#if !MESHTASTIC_EXCLUDE_POSITIONDB
s_decodePositionsTarget = nullptr;
#endif
#if !MESHTASTIC_EXCLUDE_TELEMETRYDB
s_decodeTelemetryTarget = nullptr;
#endif
#if !MESHTASTIC_EXCLUDE_ENVIRONMENTDB
s_decodeEnvironmentTarget = nullptr;
#endif
#if !MESHTASTIC_EXCLUDE_STATUSDB
s_decodeStatusTarget = nullptr;
#endif
}
/** The current change # for radio settings. Starts at 0 on boot and any time the radio settings
* might have changed is incremented. Allows others to detect they might now be on a new channel.
*/
@@ -1518,6 +1606,19 @@ void NodeDB::loadFromDisk()
}
#endif
// Arm the direct-into-map decode so satellite entries skip the temp vectors.
{
concurrency::LockGuard guard(&satelliteMutex);
armNodeDatabaseDecodeTargets();
}
struct Disarm {
NodeDB &self;
~Disarm() { self.disarmNodeDatabaseDecodeTargets(); }
} disarm{*this};
// Avoid push_back's power-of-2 capacity growth wasting RAM at small N.
nodeDatabase.nodes.reserve(MAX_NUM_NODES);
auto state = loadProto(nodeDatabaseFileName, getMaxNodesAllocatedSize(), sizeof(meshtastic_NodeDatabase),
&meshtastic_NodeDatabase_msg, &nodeDatabase);
if (nodeDatabase.version < DEVICESTATE_MIN_VER) {
@@ -1531,50 +1632,33 @@ void NodeDB::loadFromDisk()
} else {
meshNodes = &nodeDatabase.nodes;
numMeshNodes = nodeDatabase.nodes.size();
// Hydrate the satellite maps; the on-disk vectors stay empty in steady
// state and are repopulated only at save time.
concurrency::LockGuard guard(&satelliteMutex);
// Counts computed outside LOG_INFO() so cppcheck doesn't choke on #if in macro args.
const unsigned posCount =
#if !MESHTASTIC_EXCLUDE_POSITIONDB
nodePositions.clear();
nodePositions.reserve(nodeDatabase.positions.size());
for (const auto &entry : nodeDatabase.positions) {
if (entry.has_position)
nodePositions[entry.num] = entry.position;
}
nodeDatabase.positions.clear();
nodeDatabase.positions.shrink_to_fit();
(unsigned)nodePositions.size();
#else
0u;
#endif
const unsigned telCount =
#if !MESHTASTIC_EXCLUDE_TELEMETRYDB
nodeTelemetry.clear();
nodeTelemetry.reserve(nodeDatabase.telemetry.size());
for (const auto &entry : nodeDatabase.telemetry) {
if (entry.has_device_metrics)
nodeTelemetry[entry.num] = entry.device_metrics;
}
nodeDatabase.telemetry.clear();
nodeDatabase.telemetry.shrink_to_fit();
(unsigned)nodeTelemetry.size();
#else
0u;
#endif
const unsigned envCount =
#if !MESHTASTIC_EXCLUDE_ENVIRONMENTDB
nodeEnvironment.clear();
nodeEnvironment.reserve(nodeDatabase.environment.size());
for (const auto &entry : nodeDatabase.environment) {
if (entry.has_environment_metrics)
nodeEnvironment[entry.num] = entry.environment_metrics;
}
nodeDatabase.environment.clear();
nodeDatabase.environment.shrink_to_fit();
(unsigned)nodeEnvironment.size();
#else
0u;
#endif
const unsigned statusCount =
#if !MESHTASTIC_EXCLUDE_STATUSDB
nodeStatus.clear();
nodeStatus.reserve(nodeDatabase.status.size());
for (const auto &entry : nodeDatabase.status) {
if (entry.has_status)
nodeStatus[entry.num] = entry.status;
}
nodeDatabase.status.clear();
nodeDatabase.status.shrink_to_fit();
(unsigned)nodeStatus.size();
#else
0u;
#endif
LOG_INFO("Loaded saved nodedatabase version %d, with nodes count: %d", nodeDatabase.version, nodeDatabase.nodes.size());
LOG_INFO("Loaded saved nodedatabase v%d: %d nodes, %u pos, %u tel, %u env, %u status", nodeDatabase.version,
nodeDatabase.nodes.size(), posCount, telCount, envCount, statusCount);
}
if (numMeshNodes > MAX_NUM_NODES) {
@@ -1852,6 +1936,15 @@ bool NodeDB::saveNodeDatabaseToDisk()
return false;
}
// Defer (don't fail) while xmodem holds the prefs file handle. Returning false
// would propagate through saveToDisk() and trigger fsFormat() mid-transfer.
#ifdef FSCom
if (xModem.isBusy()) {
LOG_DEBUG("Deferring NodeDB save: xmodem transfer in progress");
return true;
}
#endif
#ifdef FSCom
spiLock->lock();
FSCom.mkdir("/prefs");
src/mesh/NodeDB.h
+12 -7
View File
@@ -4,9 +4,9 @@
#include <Arduino.h>
#include <algorithm>
#include <assert.h>
#include <map>
#include <pb_encode.h>
#include <string>
#include <unordered_map>
#include <vector>
#include "MeshTypes.h"
@@ -170,19 +170,19 @@ class NodeDB
Observable<const meshtastic::NodeStatus *> newStatus;
pb_size_t numMeshNodes;
// Satellite per-NodeNum maps for data we used to inline into NodeInfoLite,
// gated by MESHTASTIC_EXCLUDE_*DB so STM32WL can omit them.
// Satellite per-NodeNum maps. std::map avoids unordered_map's bucket-array
// preallocation; O(log N) lookup is fine at these sizes.
#if !MESHTASTIC_EXCLUDE_POSITIONDB
std::unordered_map<NodeNum, meshtastic_PositionLite> nodePositions;
std::map<NodeNum, meshtastic_PositionLite> nodePositions;
#endif
#if !MESHTASTIC_EXCLUDE_TELEMETRYDB
std::unordered_map<NodeNum, meshtastic_DeviceMetrics> nodeTelemetry;
std::map<NodeNum, meshtastic_DeviceMetrics> nodeTelemetry;
#endif
#if !MESHTASTIC_EXCLUDE_ENVIRONMENTDB
std::unordered_map<NodeNum, meshtastic_EnvironmentMetrics> nodeEnvironment;
std::map<NodeNum, meshtastic_EnvironmentMetrics> nodeEnvironment;
#endif
#if !MESHTASTIC_EXCLUDE_STATUSDB
std::unordered_map<NodeNum, meshtastic_StatusMessage> nodeStatus;
std::map<NodeNum, meshtastic_StatusMessage> nodeStatus;
#endif
bool keyIsLowEntropy = false;
@@ -429,6 +429,11 @@ class NodeDB
// the legacy descriptor and copies entries into the v25 layout. Caller
// is responsible for save / install-default on the result.
bool migrateLegacyNodeDatabase();
// Route satellite-store decode entries straight into our maps instead of
// temp vectors. Must be paired — disarm before any other NodeDatabase decode.
void armNodeDatabaseDecodeTargets();
void disarmNodeDatabaseDecodeTargets();
};
extern NodeDB *nodeDB;
src/mesh/mesh-pb-constants.h
+1 -1
View File
@@ -80,7 +80,7 @@ static_assert(sizeof(meshtastic_NodeInfoLite) <= 130, "NodeInfoLite size increas
#if defined(ARCH_STM32WL)
#define MAX_NUM_NODES 10
#elif defined(ARCH_NRF52)
#define MAX_NUM_NODES 80
#define MAX_NUM_NODES 150
#elif defined(CONFIG_IDF_TARGET_ESP32S3)
#include "Esp.h"
static inline int get_max_num_nodes()