Compare commits

..
1 Commits
647 changed files with 12064 additions and 53753 deletions
-20
View File
@@ -1,20 +0,0 @@
{
"version": "0.0.1",
"configurations": [
{
"name": "fleetsuite-ui",
"runtimeExecutable": "npm",
"runtimeArgs": [
"--prefix",
"mcp-server/web-ui",
"run",
"dev",
"--",
"--port",
"5199",
"--strictPort"
],
"port": 5199
}
]
}
-17
View File
@@ -1,17 +0,0 @@
{
"hooks": {
"PostToolUse": [
{
"matcher": "Write|Edit",
"hooks": [
{
"type": "command",
"command": "f=$(tr -d '\\n' | grep -o '\"file_path\"[[:space:]]*:[[:space:]]*\"[^\"]*\"' | head -1 | sed 's/.*:[[:space:]]*\"//; s/\"$//'); [ -n \"$f\" ] && [ -f \"$f\" ] || exit 0; t=$(command -v trunk || echo \"$HOME/.cache/trunk/launcher/trunk\"); [ -x \"$t\" ] || { echo \"trunk-fmt hook: trunk not found; its launcher needs curl or wget to bootstrap the CLI (see 'Formatting & the trunk toolchain' in .github/copilot-instructions.md)\" >&2; exit 1; }; out=$(\"$t\" fmt --force \"$f\" 2>&1) || { echo \"trunk-fmt hook: trunk fmt failed on $f: $out\" >&2; exit 1; }",
"timeout": 120,
"statusMessage": "Formatting (trunk)..."
}
]
}
]
}
}
+1 -1
View File
@@ -18,7 +18,7 @@ ENV PIP_ROOT_USER_ACTION=ignore
# trunk-ignore(hadolint/DL3008): apt packages are not pinned.
# trunk-ignore(terrascan/AC_DOCKER_0002): apt packages are not pinned.
RUN apt-get update && apt-get install --no-install-recommends -y \
cmake git zip libgpiod-dev libjsoncpp-dev libbluetooth-dev libi2c-dev \
cmake git zip libgpiod-dev libbluetooth-dev libi2c-dev \
libunistring-dev libmicrohttpd-dev libgnutls28-dev libgcrypt20-dev \
libusb-1.0-0-dev libssl-dev pkg-config libsqlite3-dev libsdl2-dev && \
apt-get clean && rm -rf /var/lib/apt/lists/* && \
+1 -1
View File
@@ -31,7 +31,7 @@ cmake --install "$WORK/ulfius/$SANITIZER" --prefix /usr
cd "$SRC/firmware"
PLATFORMIO_EXTRA_SCRIPTS=$(echo -e "pre:.clusterfuzzlite/platformio-clusterfuzzlite-pre.py\npost:.clusterfuzzlite/platformio-clusterfuzzlite-post.py")
STATIC_LIBS=$(pkg-config --libs --static libulfius openssl libgpiod yaml-cpp jsoncpp bluez --silence-errors)
STATIC_LIBS=$(pkg-config --libs --static libulfius openssl libgpiod yaml-cpp bluez --silence-errors)
export PLATFORMIO_EXTRA_SCRIPTS
export STATIC_LIBS
export PLATFORMIO_WORKSPACE_DIR="$WORK/pio/$SANITIZER"
-1
View File
@@ -16,7 +16,6 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
libssl-dev \
libulfius-dev \
libyaml-cpp-dev \
libjsoncpp-dev \
pipx \
pkg-config \
python3 \
+3 -7
View File
@@ -8,21 +8,17 @@
"features": {
"ghcr.io/devcontainers/features/python:1": {
"installTools": true,
"version": "3.13"
"version": "3.14"
}
},
"customizations": {
"vscode": {
"extensions": [
"ms-vscode.cpptools",
"Jason2866.esp-decoder",
"pioarduino.pioarduino-ide",
"platformio.platformio-ide",
"Trunk.io"
],
"unwantedRecommendations": [
"ms-azuretools.vscode-docker",
"platformio.platformio-ide"
],
"unwantedRecommendations": ["ms-azuretools.vscode-docker"],
"settings": {
"extensions.ignoreRecommendations": true
}
+1 -1
View File
@@ -13,7 +13,7 @@ runs:
shell: bash
run: |
sudo apt-get -y update --fix-missing
sudo apt-get install -y cppcheck libbluetooth-dev libgpiod-dev libyaml-cpp-dev libjsoncpp-dev lsb-release
sudo apt-get install -y cppcheck libbluetooth-dev libgpiod-dev libyaml-cpp-dev lsb-release
- name: Setup Python
uses: actions/setup-python@v6
+1 -1
View File
@@ -11,4 +11,4 @@ runs:
- name: Install libs needed for native build
shell: bash
run: |
sudo apt-get install -y libbluetooth-dev libgpiod-dev libyaml-cpp-dev libjsoncpp-dev openssl libssl-dev libulfius-dev liborcania-dev libusb-1.0-0-dev libi2c-dev libuv1-dev
sudo apt-get install -y libbluetooth-dev libgpiod-dev libyaml-cpp-dev openssl libssl-dev libulfius-dev liborcania-dev libusb-1.0-0-dev libi2c-dev libuv1-dev
+8 -125
View File
@@ -135,99 +135,6 @@ On top of authorization, any remote admin message that **mutates** state (not a
- **Channel 0 PSK change** → every peer must re-learn the channel hash; cached NodeInfo becomes temporarily unreachable until the next broadcast.
- **`security.private_key` blanked via admin** → regenerates both halves (unless in Ham mode) and propagates the new public key via NodeInfo.
## NodeDB Layout (v25)
`DEVICESTATE_CUR_VER = 25`, `DEVICESTATE_MIN_VER = 24`. The on-device NodeDB was split in v25 into a slim header table plus four optional satellite stores. Older v24 saves auto-migrate at boot. Old training-data instincts (`node->user.long_name`, `node->position.latitude_i`, `node->is_favorite`, `node->device_metrics.battery_level`) are wrong now — the fields aren't there. Read this section before touching anything that walks `nodeDB->meshNodes`.
### Slim `NodeInfoLite`
`UserLite` is flattened onto `NodeInfoLite` (no nested sub-message); `position` and `device_metrics` are removed entirely (tags reserved). MAC address is dropped. Long names are capped at 25 chars (`max_size:25` in `deviceonly.options`); `hw_model` and `role` are `int_size:8`. Encoded size dropped from ~166 B → ~105 B per node.
Booleans are bit-packed into `NodeInfoLite.bitfield`. **Do not read or write the bits directly** — use the inline helpers in `src/mesh/NodeDB.h`:
```cpp
nodeInfoLiteHasUser(n) // bit 5 — user fields populated
nodeInfoLiteIsFavorite(n) // bit 3
nodeInfoLiteIsIgnored(n) // bit 4
nodeInfoLiteIsMuted(n) // bit 1
nodeInfoLiteIsLicensed(n) // bit 6 — Ham mode peer
nodeInfoLiteIsKeyManuallyVerified(n) // bit 0
nodeInfoLiteHasIsUnmessagable(n) // bit 8 — "is_unmessagable was sent"
nodeInfoLiteIsUnmessagable(n) // bit 7
// via_mqtt is bit 2 (mask exposed; predicate uses the mask directly)
nodeInfoLiteSetBit(n, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true); // setter
```
### Satellite stores
Four `std::unordered_map<NodeNum, …>` members on `NodeDB`, each gated by its own build flag:
| Map | Value type | Build flag |
| ----------------- | ------------------------------- | ---------------------------------- |
| `nodePositions` | `meshtastic_PositionLite` | `MESHTASTIC_EXCLUDE_POSITIONDB` |
| `nodeTelemetry` | `meshtastic_DeviceMetrics` | `MESHTASTIC_EXCLUDE_TELEMETRYDB` |
| `nodeEnvironment` | `meshtastic_EnvironmentMetrics` | `MESHTASTIC_EXCLUDE_ENVIRONMENTDB` |
| `nodeStatus` | `meshtastic_StatusMessage` | `MESHTASTIC_EXCLUDE_STATUSDB` |
Defaults are ON (i.e., maps **excluded**) for STM32WL only — see `src/mesh/mesh-pb-constants.h`. On every other arch all four maps are present. When excluded, the map member is absent and the corresponding accessors return `false`.
All four maps are guarded by **`mutable concurrency::Lock satelliteMutex`** — concurrent access from receive threads, the phone API state machine, and the renderer is the rule, not the exception.
### Accessor convention
**Never hand out pointers into the maps.** Use the copy-out accessors on `NodeDB`:
```cpp
bool copyNodePosition(NodeNum, meshtastic_PositionLite &out) const;
bool copyNodeTelemetry(NodeNum, meshtastic_DeviceMetrics &out) const;
bool copyNodeEnvironment(NodeNum, meshtastic_EnvironmentMetrics &out) const;
bool copyNodeStatus(NodeNum, meshtastic_StatusMessage &out) const;
```
Each takes the lock, copies the value if present, returns `false` if the entry is absent or the DB is excluded. Pass-by-out-param is deliberate — pointer-style accessors would invite UAF and lock-leak bugs across the renderer. The "has any X" convenience predicates (`hasValidPosition` etc.) are implemented in terms of these.
Writers go through `setNodeStatus`, `updatePosition`, `updateTelemetry` (which dispatches on `which_variant` for device vs environment metrics) — these own the lock and the eviction hooks.
### Eviction
Every code path that drops a node from the header table must also evict the satellites. The single chokepoint is `eraseNodeSatellites(NodeNum)`; it's already called from `getOrCreateMeshNode`'s oldest-boring eviction, `removeNodeByNum`, both branches of `resetNodes`, `cleanupMeshDB`, `addFromContact`'s ignored-branch, and `AdminModule`'s `set_ignored_node`. Add new eviction sites here, not by calling `.erase()` directly.
### Sync flow: thin NodeInfo + post-COMPLETE_ID replay (no opt-in)
There is no capability flag and no special "gradient" nonce. The **default** sync flow is:
1. Config / module-config / channel / metadata segments (same as before).
2. `STATE_SEND_OWN_NODEINFO`**our own** NodeInfo, still bundled with our position and device_metrics (because the replay snapshot excludes our own NodeNum). Emitted via `ConvertToNodeInfo(lite)`.
3. `STATE_SEND_OTHER_NODEINFOS` — every other peer's NodeInfo, **always thin** (no `position`, no `device_metrics`). Emitted via `ConvertToNodeInfoThin(lite)`.
4. `STATE_SEND_FILEMANIFEST``STATE_SEND_COMPLETE_ID` — the phone sees `config_complete_id` and treats sync as done.
5. `STATE_SEND_PACKETS` — live mesh packets, with a trailing replay drain interleaved. The replay drain walks four cached satellite stores in order (positions → telemetry → environment → status) and emits each cached entry as an ordinary `MeshPacket` on the matching portnum (`POSITION_APP`, `TELEMETRY_APP` device + environment variants, `NODE_STATUS_APP`). These are indistinguishable on the wire from live mesh traffic, so clients need no special handling — any code that already updates UI on `POSITION_APP` etc. works.
`PhoneAPI::sendConfigComplete()` arms `replayPhase = REPLAY_PHASE_POSITIONS` for default/full sync and `SPECIAL_NONCE_ONLY_NODES`, while `SPECIAL_NONCE_ONLY_CONFIG` skips replay. The drain runs inside `STATE_SEND_PACKETS` via `popReplayPacket()`, lower priority than live traffic. When all four phases drain, `replayPhase` flips back to `REPLAY_PHASE_IDLE` and the snapshot vectors get `shrink_to_fit`ed.
STM32WL and any other build with all four `MESHTASTIC_EXCLUDE_*DB` flags set produces zero replay packets — `popReplayPacket` advances through each phase in microseconds without emitting anything.
Special nonces that still mean something:
- `SPECIAL_NONCE_ONLY_CONFIG` (69420) — skip node sync entirely, just config.
- `SPECIAL_NONCE_ONLY_NODES` (69421) — skip config segments, jump straight to `STATE_SEND_OWN_NODEINFO`. Still gets the post-COMPLETE_ID replay drain.
There are no other reserved nonces; everything else is a fresh random `want_config_id` from the client.
### v24 → v25 migration
The legacy migration code lives in **`src/mesh/NodeDBLegacyMigration.cpp`**, not in `NodeDB.cpp`. It owns the `meshtastic_NodeDatabase_Legacy` callback and `NodeDB::migrateLegacyNodeDatabase()`. The legacy proto descriptor is `protobufs/meshtastic/deviceonly_legacy.proto` (only included by the migration TU). The boot path peeks the file's leading version tag, runs the migration if `version < 25`, then re-saves in v25 layout. The legacy descriptor is scheduled for removal once `DEVICESTATE_MIN_VER` is bumped.
### Read-site rules of thumb
- Never `node->position.X` / `node->device_metrics.X` — those fields no longer exist. Pull from the satellite map via `copyNodePosition` / `copyNodeTelemetry`.
- Never `node->user.long_name``long_name`, `short_name`, `public_key`, `hw_model`, `role`, `macaddr` (gone), `is_licensed`, `is_unmessagable` are flat on `NodeInfoLite`.
- Never `node->is_favorite` / `node->is_ignored` / `node->via_mqtt` / `node->is_key_manually_verified` — use the bitfield helpers.
- Never assume `nodeDB->getMeshNode(num)->position.time` — call `copyNodePosition` and check the return.
- Don't lock `satelliteMutex` yourself in renderer code; the copy-out accessors already do.
Unit tests for the conversion layer live in `test/test_type_conversions/test_main.cpp` (Unity) — bitfield round-trips, `long_name` truncation, thin-vs-full conversions. Add cases there when extending the schema.
## Project Structure
```
@@ -283,15 +190,6 @@ firmware/
## Coding Conventions
### Formatting & the trunk toolchain
`trunk fmt` is the project formatter (`trunk_check` CI rejects unformatted code). For Claude Code users, `.claude/settings.json` ships a PostToolUse hook that runs `trunk fmt --force` on every file the agent writes or edits. The hook is pure sh/grep/sed — no python or jq required — but trunk itself must be able to run:
- Trunk's launcher (`~/.cache/trunk/launcher/trunk`, or `trunk` on PATH) downloads the CLI version pinned in `.trunk/trunk.yaml` on first use and again whenever that pin is bumped. **The launcher needs `curl` or `wget`**; without one it fails with "Cannot download… please install curl or wget", and the hook surfaces that as a warning on every write.
- No curl/wget available (e.g. a minimal WSL image)? Bootstrap by hand with any Python (PlatformIO bundles one at `~/.platformio/penv/bin/python`): download `https://trunk.io/releases/<ver>/trunk-<ver>-linux-x86_64.tar.gz` and place the `trunk` binary at `~/.cache/trunk/cli/<ver>-linux-x86_64/trunk` (chmod +x), where `<ver>` is the `cli.version` from `.trunk/trunk.yaml`.
- The hook fails loudly by design (visible warning, non-blocking). Silent no-op formatting hooks hide real breakage — don't re-add `2>/dev/null || true` around the whole thing.
- More generally: don't assume a stock Linux userland in hooks or helper scripts — minimal WSL/container images may lack `python3`, `curl`, `wget`, and `jq`. Prefer plain sh + coreutils, or PlatformIO's bundled Python for anything heavier.
### General Style
- Follow existing code style - run `trunk fmt` before commits
@@ -618,35 +516,20 @@ Most workflows can be triggered manually via `workflow_dispatch` for testing.
### Native unit tests (C++)
Unit tests in `test/` directory with 17 test suites:
Unit tests in `test/` directory with 12 test suites:
- `test_admin_radio/` - LoRa region/config validation and AdminModule dispatch
- `test_atak/` - ATAK integration
- `test_crypto/` - Cryptography
- `test_default/` - Default configuration
- `test_http_content_handler/` - HTTP handling
- `test_mac_from_string/` - MAC address parsing
- `test_mqtt/` - MQTT integration
- `test_radio/` - Radio interface
- `test_mesh_module/` - Module framework
- `test_meshpacket_serializer/` - Packet serialization
- `test_mqtt/` - MQTT integration
- `test_packet_history/` - Packet history tracking
- `test_position_precision/` - Position precision helpers
- `test_radio/` - Radio interface
- `test_serial/` - Serial communication
- `test_traffic_management/` - Traffic management
- `test_transmit_history/` - Retransmission tracking
- `test_type_conversions/` - NodeDB v25 type conversion (bitfield round-trips, NodeInfoLite)
- `test_utf8/` - UTF-8 utilities
- `test_atak/` - ATAK integration
- `test_default/` - Default configuration
- `test_http_content_handler/` - HTTP handling
- `test_serial/` - Serial communication
Run command (preferred — avoids pipe-buffering and the Ubuntu externally-managed-environment error):
```bash
~/.platformio/penv/bin/python -m platformio test -e native -f test_your_suite > /tmp/test_out.txt 2>&1
grep -E 'error:|PASS|FAIL|succeeded|failed' /tmp/test_out.txt
tail -15 /tmp/test_out.txt
```
Do **not** use `pio test … | tail -N` — it discards build errors and shows stale cached results. Do **not** use `pio test … | grep` — line-buffering makes the terminal appear hung until the process exits. Redirect to a file first, then grep.
Run with: `pio test -e native`
Simulation testing: `bin/test-simulator.sh`
+25 -39
View File
@@ -4,14 +4,9 @@ on:
workflow_dispatch:
inputs:
# trunk-ignore(checkov/CKV_GHA_7)
target:
type: string
required: false
description: Choose the target board, e.g. nrf52_promicro_diy_tcxo. If blank, will find available targets.
arch:
type: choice
options:
- all
- esp32
- esp32s3
- esp32c3
@@ -20,18 +15,32 @@ on:
- rp2040
- rp2350
- stm32
description: Choose an arch to limit the search, or 'all' to search all architectures.
default: all
target:
type: string
required: false
description: Choose the target board, e.g. nrf52_promicro_diy_tcxo. If blank, will find available targets.
# find-target:
# type: boolean
# default: true
# description: 'Find the available targets'
permissions: read-all
jobs:
find-targets:
if: ${{ inputs.target == '' }}
strategy:
fail-fast: false
matrix:
arch:
- all
- esp32
- esp32s3
- esp32c3
- esp32c6
- nrf52840
- rp2040
- rp2350
- stm32
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v6
@@ -42,37 +51,14 @@ jobs:
- run: pip install -U platformio
- name: Generate matrix
id: jsonStep
env:
BUILDTARGET: ${{ inputs.target }}
MATRIXARCH: ${{ inputs.arch }}
run: |
TARGETS=$(./bin/generate_ci_matrix.py ${{matrix.arch}} --level extra)
if [ "$BUILDTARGET" = "" ]; then
echo "Name: $GITHUB_REF_NAME" >> $GITHUB_STEP_SUMMARY
echo "Base: $GITHUB_BASE_REF" >> $GITHUB_STEP_SUMMARY
echo "Arch: $MATRIXARCH" >> $GITHUB_STEP_SUMMARY
echo "Ref: $GITHUB_REF" >> $GITHUB_STEP_SUMMARY
echo "## 🎯 The following target boards are available to build:" >> $GITHUB_STEP_SUMMARY
echo "| Platform | Board |" >> $GITHUB_STEP_SUMMARY
echo "| -------- | ----- |" >> $GITHUB_STEP_SUMMARY
echo $TARGETS | jq -r 'sort_by(.board) | sort_by(.platform) |.[] | "| " + .platform + " | " + .board + " |" ' >> $GITHUB_STEP_SUMMARY
else
echo "We build this one:" >> $GITHUB_STEP_SUMMARY
ARCH=$(echo "$TARGETS" | jq --arg BUILDTARGET "$BUILDTARGET" -r '.[] | select(.board==$BUILDTARGET) | .platform')
echo "| Platform | Board |" >> $GITHUB_STEP_SUMMARY
echo "| -------- | ----- |" >> $GITHUB_STEP_SUMMARY
echo "| $ARCH | "$BUILDTARGET" |" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
if [[ "$ARCH" == "" ]]; then
echo "## ❌ Error: Target "$BUILDTARGET" not found!" >> $GITHUB_STEP_SUMMARY
else
echo "## ✅ Target "$BUILDTARGET" found, proceeding to build." >> $GITHUB_STEP_SUMMARY
fi
echo "You may need to refresh this page to make the built firmware appear below." >> $GITHUB_STEP_SUMMARY
echo "arch=$ARCH" >> $GITHUB_OUTPUT
fi
outputs:
arch: ${{ steps.jsonStep.outputs.arch }}
echo "Name: $GITHUB_REF_NAME" >> $GITHUB_STEP_SUMMARY
echo "Base: $GITHUB_BASE_REF" >> $GITHUB_STEP_SUMMARY
echo "Arch: ${{matrix.arch}}" >> $GITHUB_STEP_SUMMARY
echo "Ref: $GITHUB_REF" >> $GITHUB_STEP_SUMMARY
echo "Targets:" >> $GITHUB_STEP_SUMMARY
echo $TARGETS | jq -r 'sort_by(.board) |.[] | "- " + .board' >> $GITHUB_STEP_SUMMARY
version:
if: ${{ inputs.target != '' }}
@@ -92,12 +78,12 @@ jobs:
build:
if: ${{ inputs.target != '' && inputs.arch != 'native' }}
needs: [version, find-targets]
needs: [version]
uses: ./.github/workflows/build_firmware.yml
with:
version: ${{ needs.version.outputs.long }}
pio_env: ${{ inputs.target }}
platform: ${{ needs.find-targets.outputs.arch }}
platform: ${{ inputs.arch }}
gather-artifacts:
permissions:
@@ -1,62 +0,0 @@
name: Post Firmware Size Comment
on:
workflow_run:
workflows: [CI]
types: [completed]
permissions:
pull-requests: write
actions: read
jobs:
post-size-comment:
if: >
github.event.workflow_run.event == 'pull_request' &&
github.event.workflow_run.conclusion != 'cancelled' &&
github.repository == 'meshtastic/firmware'
continue-on-error: true
runs-on: ubuntu-latest
steps:
- name: Download size report
id: download
uses: actions/download-artifact@v8
continue-on-error: true
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
run-id: ${{ github.event.workflow_run.id }}
name: size-report
path: ./
- name: Post or update PR comment
if: steps.download.outcome == 'success'
uses: actions/github-script@v8
with:
script: |
const fs = require('fs');
const marker = '<!-- firmware-size-report -->';
const body = fs.readFileSync('./size-report.md', 'utf8');
const prNumber = parseInt(fs.readFileSync('./pr-number.txt', 'utf8').trim(), 10);
const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
});
const existing = comments.find(c => c.body.includes(marker));
if (existing) {
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: existing.id,
body,
});
} else {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
body,
});
}
-187
View File
@@ -1,187 +0,0 @@
name: Post Web Flasher Link Comment
on:
workflow_run:
workflows: [CI]
types: [completed]
permissions:
pull-requests: write
actions: read
jobs:
post-flasher-link:
if: >
github.event.workflow_run.event == 'pull_request' &&
github.event.workflow_run.conclusion != 'cancelled' &&
github.repository == 'meshtastic/firmware'
continue-on-error: true
runs-on: ubuntu-latest
steps:
# Per-board manifests carry the firmware's own metadata (activelySupported,
# displayName, ...) generated from each target's custom_meshtastic_* config.
- name: Download board manifests
uses: actions/download-artifact@v8
continue-on-error: true
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
run-id: ${{ github.event.workflow_run.id }}
pattern: manifest-*
path: ./manifests
merge-multiple: true
- name: Post or update web flasher link comment
uses: actions/github-script@v8
with:
script: |
const marker = '<!-- web-flasher-link -->';
const run = context.payload.workflow_run;
const { owner, repo } = context.repo;
// Resolve the PR by matching the run's head SHA against the repo's open
// PRs. workflow_run.pull_requests is empty for fork PRs, and
// listPullRequestsAssociatedWithCommit won't return an open fork PR by
// its head commit — but pulls.list includes fork PRs. Matching on head
// SHA also enforces that the run is for the PR's current commit, so stale
// re-runs of an outdated commit won't match.
const openPrs = await github.paginate(github.rest.pulls.list, {
owner, repo, state: 'open', per_page: 100,
});
const pr = openPrs.find((p) => p.head.sha === run.head_sha);
if (!pr) {
core.info(`No open pull request matches commit ${run.head_sha}; skipping.`);
return;
}
const prNumber = pr.number;
// Restrict to trusted authors. NOTE: author_association is computed for
// the GITHUB_TOKEN, which cannot see *private/concealed* org memberships —
// those members come back as CONTRIBUTOR, not MEMBER. So gating on MEMBER
// alone silently excludes most maintainers. We allow the trusted set the
// token can actually identify (members, collaborators, and anyone with a
// previously merged PR). For strict members-only you'd need an org-read
// App/PAT token to call orgs.checkMembershipForUser.
const allowedAssociations = ['OWNER', 'MEMBER', 'COLLABORATOR', 'CONTRIBUTOR'];
if (!allowedAssociations.includes(pr.author_association)) {
core.info(`Author association ${pr.author_association} is not trusted; skipping.`);
return;
}
// Require at least one per-arch firmware artifact from gather-artifacts
const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, {
owner, repo, run_id: run.id, per_page: 100,
});
const archRe = /^firmware-(esp32|esp32s3|esp32c3|esp32c6|nrf52840|rp2040|rp2350|stm32)-(\d+\.\d+\.\d+\.[0-9a-f]+)$/;
const archArtifacts = artifacts.filter((a) => archRe.test(a.name) && !a.expired);
if (archArtifacts.length === 0) {
core.info('No per-arch firmware artifacts found; skipping.');
return;
}
const version = archRe.exec(archArtifacts[0].name)[2];
const expiresAt = archArtifacts[0].expires_at
? new Date(archArtifacts[0].expires_at).toISOString().slice(0, 10)
: null;
// Read each built board's manifest (.mt.json). activelySupported,
// displayName and architecture come straight from the board's
// custom_meshtastic_* platformio config, so the list is in sync with
// the firmware itself — no external device database needed.
const fs = require('fs');
let boards = [];
try {
boards = fs.readdirSync('./manifests')
.filter((f) => f.endsWith('.mt.json'))
.map((f) => {
try { return JSON.parse(fs.readFileSync(`./manifests/${f}`, 'utf8')); }
catch { return null; }
})
.filter((m) => m && m.activelySupported === true && m.platformioTarget)
.map((m) => ({
board: m.platformioTarget,
platform: m.architecture || '',
// displayName is maintainer-authored text; escape table-breaking pipes
displayName: String(m.displayName || m.platformioTarget).replace(/\|/g, '\\|'),
image: Array.isArray(m.images) && m.images[0] ? String(m.images[0]) : '',
}))
.sort((a, b) => a.board.localeCompare(b.board));
} catch (e) {
core.warning(`Could not read board manifests: ${e.message}`);
}
const flasherUrl = `https://flasher.meshtastic.org/?pr=${prNumber}`;
// Device illustrations are served by the flasher from the same image
// names the manifest declares (custom_meshtastic_images). The flasher
// serves its SPA shell (HTML, 200) for unknown paths, so confirm each
// image really resolves to an image before linking it.
const imageBase = 'https://flasher.meshtastic.org/img/devices/';
await Promise.all(boards.map(async (b) => {
if (!b.image) return;
try {
const res = await fetch(`${imageBase}${encodeURIComponent(b.image)}`);
const type = res.headers.get('content-type') || '';
if (!res.ok || !type.startsWith('image/')) b.image = '';
} catch { b.image = ''; }
}));
const boardLines = boards
.map((b) => {
const img = b.image ? `<img src="${imageBase}${encodeURIComponent(b.image)}" alt="" height="34">` : '';
return `| ${img} | ${b.displayName} | [\`${b.board}\`](${flasherUrl}&device=${encodeURIComponent(b.board)}) | ${b.platform} |`;
})
.join('\n');
// Shields.io badges. Only non-user-controlled, charset-constrained values
// (version, commit sha, counts, dates) go into badge URLs — never board
// names or the PR title — so the rendered comment cannot be spoofed.
const shieldText = (s) =>
encodeURIComponent(String(s).replace(/-/g, '--').replace(/_/g, '__').replace(/ /g, '_'));
const shield = (label, message, color) =>
`https://img.shields.io/badge/${shieldText(label)}-${shieldText(message)}-${color}`;
const buttonUrl =
`https://img.shields.io/badge/${shieldText('Flash this PR in the Web Flasher')}-2C2D3C?style=for-the-badge`;
const badges = [
`![firmware](${shield('firmware', version, '67EA94')})`,
`![commit](${shield('commit', run.head_sha.slice(0, 7), '2C2D3C')})`,
`![boards](${shield('boards', boards.length, '5C6BC0')})`,
];
if (expiresAt) badges.push(`![expires](${shield('expires', expiresAt, '9A4E00')})`);
// Only render the board table when there are supported boards to list
const boardTable = boards.length > 0 ? [
`<details><summary>Supported boards built by this PR (${boards.length})</summary>`,
'',
'| | Device | Board | Platform |',
'| --- | --- | --- | --- |',
boardLines,
'',
'</details>',
'',
] : [];
const body = [
marker,
'## ⚡ Try this PR in the Web Flasher',
'',
`[![Flash this PR in the Web Flasher](${buttonUrl})](${flasherUrl})`,
'',
badges.join(' '),
'',
'> [!WARNING]',
'> This is an automated, unreviewed CI test build. Back up your device configuration',
'> before flashing, and only flash devices you are able to recover.',
'',
...boardTable,
`*Build artifacts expire${expiresAt ? ` on ${expiresAt}` : ' after 30 days'}. Updated for \`${run.head_sha.slice(0, 7)}\`.*`,
].join('\n');
// Sticky comment: update in place when the marker is found
const comments = await github.paginate(github.rest.issues.listComments, {
owner, repo, issue_number: prNumber, per_page: 100,
});
const existing = comments.find((c) => c.body?.includes(marker));
if (existing) {
await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body });
} else {
await github.rest.issues.createComment({ owner, repo, issue_number: prNumber, body });
}
@@ -1,62 +0,0 @@
name: Post Web Flasher Build Placeholder
# Drops an immediate "build in progress" comment when a PR opens, so the web
# flasher entry shows up right away. The real CI-driven workflow
# (flasher-link-comment.yml) later replaces it in place via the shared marker.
#
# SECURITY: this uses pull_request_target (write token, runs for fork PRs) but is
# safe because it never checks out or runs PR code and posts a fully static body
# — no PR title, branch name, or other untrusted input is used anywhere.
on:
pull_request_target:
types: [opened, reopened]
permissions:
pull-requests: write
jobs:
post-placeholder:
if: github.repository == 'meshtastic/firmware'
continue-on-error: true
runs-on: ubuntu-latest
steps:
- name: Post web flasher build-in-progress placeholder
uses: actions/github-script@v8
with:
script: |
const marker = '<!-- web-flasher-link -->';
const { owner, repo } = context.repo;
const pr = context.payload.pull_request;
// Trusted authors only (matches the real workflow). author_association
// can't reflect private org membership for the token, so concealed
// members appear as CONTRIBUTOR — include it, or maintainers are excluded.
const allowedAssociations = ['OWNER', 'MEMBER', 'COLLABORATOR', 'CONTRIBUTOR'];
if (!allowedAssociations.includes(pr.author_association)) {
core.info(`Author association ${pr.author_association} is not trusted; skipping.`);
return;
}
// Only seed a placeholder when no flasher comment exists yet — never
// overwrite a real (or existing placeholder) comment.
const comments = await github.paginate(github.rest.issues.listComments, {
owner, repo, issue_number: pr.number, per_page: 100,
});
if (comments.some((c) => c.body?.includes(marker))) {
core.info('Flasher comment already exists; nothing to do.');
return;
}
const body = [
marker,
'## ⚡ Try this PR in the Web Flasher',
'',
'> [!NOTE]',
'> Building this pull request… the flash button, badges and supported-board',
'> list will appear here automatically once CI finishes.',
].join('\n');
await github.rest.issues.createComment({
owner, repo, issue_number: pr.number, body,
});
+28 -109
View File
@@ -82,9 +82,8 @@ jobs:
fail-fast: false
matrix:
check: ${{ fromJson(needs.setup.outputs.check) }}
# Runs on GitHub-hosted runners so checks don't compete with builds for the
# self-hosted 'arctastic' pool (which builds use).
runs-on: ubuntu-latest
# Use 'arctastic' self-hosted runner pool when checking in the main repo
runs-on: ${{ github.repository_owner == 'meshtastic' && 'arctastic' || 'ubuntu-latest' }}
if: ${{ github.event_name != 'workflow_dispatch' && github.repository == 'meshtastic/firmware' }}
steps:
- uses: actions/checkout@v6
@@ -246,126 +245,47 @@ jobs:
path: ./*.elf
retention-days: 30
firmware-size-report:
shame:
if: github.repository == 'meshtastic/firmware'
continue-on-error: true
permissions:
contents: read
actions: read
runs-on: ubuntu-latest
needs: [build]
steps:
- uses: actions/checkout@v6
- name: Download current manifests
if: github.event_name == 'pull_request'
with:
filter: blob:none # means we download all the git history but none of the commit (except ones with checkout like the head)
fetch-depth: 0
- name: Download the current manifests
uses: actions/download-artifact@v8
with:
path: ./manifests/
path: ./manifests-new/
pattern: manifest-*
merge-multiple: true
- name: Collect current firmware sizes
run: python3 bin/collect_sizes.py ./manifests/ ./current-sizes.json
- name: Upload size report artifact
- name: Upload combined manifests for later commit and global stats crunching.
uses: actions/upload-artifact@v7
id: upload-manifest
with:
name: firmware-sizes-${{ github.sha }}
name: manifests-${{ github.sha }}
overwrite: true
path: ./current-sizes.json
retention-days: 90
- name: Download baseline sizes from develop
path: manifests-new/*.mt.json
- name: Find the merge base
if: github.event_name == 'pull_request'
continue-on-error: true
id: baseline-develop
run: echo "MERGE_BASE=$(git merge-base "origin/$base" "$head")" >> $GITHUB_ENV
env:
GH_TOKEN: ${{ github.token }}
run: |
RUN_ID=$(gh run list -R "${{ github.repository }}" \
--workflow CI --branch develop --status success \
--limit 1 --json databaseId --jq '.[0].databaseId // empty')
if [ -n "$RUN_ID" ]; then
ARTIFACT_NAME=$(gh api "repos/${{ github.repository }}/actions/runs/${RUN_ID}/artifacts" \
--jq '.artifacts[] | select(.name | startswith("firmware-sizes-")) | select(.expired == false) | .name' | head -1)
if [ -n "$ARTIFACT_NAME" ]; then
gh run download "$RUN_ID" -R "${{ github.repository }}" \
--name "$ARTIFACT_NAME" --dir ./baseline-develop/
cp "./baseline-develop/current-sizes.json" ./develop-sizes.json
echo "found=true" >> "$GITHUB_OUTPUT"
else
echo "found=false" >> "$GITHUB_OUTPUT"
fi
else
echo "found=false" >> "$GITHUB_OUTPUT"
fi
- name: Download baseline sizes from master
if: github.event_name == 'pull_request'
continue-on-error: true
id: baseline-master
env:
GH_TOKEN: ${{ github.token }}
run: |
RUN_ID=$(gh run list -R "${{ github.repository }}" \
--workflow CI --branch master --status success \
--limit 1 --json databaseId --jq '.[0].databaseId // empty')
if [ -n "$RUN_ID" ]; then
ARTIFACT_NAME=$(gh api "repos/${{ github.repository }}/actions/runs/${RUN_ID}/artifacts" \
--jq '.artifacts[] | select(.name | startswith("firmware-sizes-")) | select(.expired == false) | .name' | head -1)
if [ -n "$ARTIFACT_NAME" ]; then
gh run download "$RUN_ID" -R "${{ github.repository }}" \
--name "$ARTIFACT_NAME" --dir ./baseline-master/
cp "./baseline-master/current-sizes.json" ./master-sizes.json
echo "found=true" >> "$GITHUB_OUTPUT"
else
echo "found=false" >> "$GITHUB_OUTPUT"
fi
else
echo "found=false" >> "$GITHUB_OUTPUT"
fi
- name: Generate size comparison report
if: github.event_name == 'pull_request'
id: report
run: |
ARGS="./current-sizes.json"
if [ -f ./develop-sizes.json ]; then
ARGS="$ARGS --baseline develop:./develop-sizes.json"
fi
if [ -f ./master-sizes.json ]; then
ARGS="$ARGS --baseline master:./master-sizes.json"
fi
REPORT=$(python3 bin/size_report.py $ARGS)
if [ -z "$REPORT" ]; then
echo "has_report=false" >> "$GITHUB_OUTPUT"
else
echo "has_report=true" >> "$GITHUB_OUTPUT"
{
echo '<!-- firmware-size-report -->'
echo '# Firmware Size Report'
echo ''
echo "$REPORT"
echo ''
echo '---'
echo "*Updated for ${{ github.sha }}*"
} > ./size-report.md
cat ./size-report.md >> "$GITHUB_STEP_SUMMARY"
fi
- name: Save PR number
if: github.event_name == 'pull_request' && steps.report.outputs.has_report == 'true'
run: echo "${{ github.event.pull_request.number }}" > ./pr-number.txt
- name: Upload size report
if: github.event_name == 'pull_request' && steps.report.outputs.has_report == 'true'
uses: actions/upload-artifact@v7
with:
name: size-report
path: |
./size-report.md
./pr-number.txt
retention-days: 5
base: ${{ github.base_ref }}
head: ${{ github.sha }}
# Currently broken (for-loop through EVERY artifact -- rate limiting)
# - name: Download the old manifests
# if: github.event_name == 'pull_request'
# run: gh run download -R "$repo" --name "manifests-$merge_base" --dir manifest-old/
# env:
# GH_TOKEN: ${{ github.token }}
# merge_base: ${{ env.MERGE_BASE }}
# repo: ${{ github.repository }}
# - name: Do scan and post comment
# if: github.event_name == 'pull_request'
# run: python3 bin/shame.py ${{ github.event.pull_request.number }} manifests-old/ manifests-new/
release-artifacts:
permissions: # Needed for 'gh release upload'.
@@ -413,7 +333,6 @@ jobs:
prerelease: true
name: Meshtastic Firmware ${{ needs.version.outputs.long }} Alpha
tag_name: v${{ needs.version.outputs.long }}
target_commitish: ${{ github.sha }}
body: ${{ steps.release_notes.outputs.notes }}
- name: Download source deb
+2 -36
View File
@@ -37,33 +37,13 @@ jobs:
sed -i -e "s#${PWD}#.#" coverage_base.info # Make paths relative.
- name: Integration test
# Cap the whole step: if the simulator ever fails to exit (e.g. the
# exit_simulator admin path regresses again) the job must fail fast,
# not run to GitHub's 6-hour limit.
timeout-minutes: 5
run: |
.pio/build/coverage/meshtasticd -s &
PID=$!
trap 'kill "$PID" 2>/dev/null || true' EXIT
timeout 20 bash -c "until ls -al /proc/$PID/fd | grep socket; do sleep 1; done"
echo "Simulator started, launching python test..."
python3 -c 'from meshtastic.test import testSimulator; testSimulator()'
# The Python harness sends exit_simulator and exits; the simulator is
# expected to terminate on its own. Give it a moment, then verify.
# If it is still alive the exit handshake is broken — fail loudly and
# do NOT fall through to `wait`, which would otherwise block until the
# job's hard timeout.
for i in $(seq 1 10); do
kill -0 "$PID" 2>/dev/null || break
sleep 1
done
if kill -0 "$PID" 2>/dev/null; then
echo "::error title=Simulator did not exit::meshtasticd ignored exit_simulator and is still running after the integration test. The exit_simulator admin path is broken (see AdminModule::handleReceivedProtobuf, ARCH_PORTDUINO bypass). Killing it to avoid a 6-hour CI overrun."
kill -9 "$PID" 2>/dev/null || true
wait "$PID" 2>/dev/null || true
exit 1
fi
wait "$PID" 2>/dev/null || true
wait
- name: Capture coverage information
if: always() # run this step even if previous step failed
@@ -106,13 +86,7 @@ jobs:
run: sed -i 's/-DBUILD_EPOCH=$UNIX_TIME/#-DBUILD_EPOCH=$UNIX_TIME/' platformio.ini
- name: PlatformIO Tests
run: |
set -o pipefail
# Filter out SKIPPED summary rows for hardware variants that can't run on the
# native host. They flood the log and make it harder to spot real failures.
# The JUnit XML is written directly to testreport.xml before the pipe, so
# the test artifact is unaffected.
platformio test -e coverage -v --junit-output-path testreport.xml 2>&1 | grep -v "[[:space:]]SKIPPED$"
run: platformio test -e coverage -v --junit-output-path testreport.xml
- name: Save test results
if: always() # run this step even if previous step failed
@@ -161,14 +135,6 @@ jobs:
name: platformio-test-report-${{ steps.version.outputs.long }}
merge-multiple: true
- name: Drop no-status testsuites from the report
# PlatformIO emits a self-closing <testsuite tests="0"/> row for every test_* dir
# crossed with every hardware variant it cannot run on the native host (~4900 rows).
# They carry no pass/fail/skip status and bury the suites that actually ran. Strip
# them so the Test Report lists only suites with a real status. Only the copy the
# reporter renders is trimmed; the uploaded artifact keeps the full XML.
run: sed -i -E 's#<testsuite [^>]*tests="0"[^>]*/>##g' testreport.xml
- name: Test Report
uses: dorny/test-reporter@v3.0.0
with:
+4 -9
View File
@@ -16,18 +16,13 @@ jobs:
submodules: true
- name: Update submodule
if: ${{ github.ref_name == 'master' || github.ref_name == 'develop' }}
working-directory: protobufs
env:
# Use the branch that triggered the workflow as the protobuf branch.
GIT_BRANCH: ${{ github.ref_name }}
if: ${{ github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop' }}
run: |
git fetch --prune origin $GIT_BRANCH
git checkout origin/$GIT_BRANCH
git submodule update --remote protobufs
- name: Download nanopb
run: |
wget https://github.com/nanopb/nanopb/releases/download/nanopb-0.4.9.1/nanopb-0.4.9.1-linux-x86.tar.gz
wget https://jpa.kapsi.fi/nanopb/download/nanopb-0.4.9.1-linux-x86.tar.gz
tar xvzf nanopb-0.4.9.1-linux-x86.tar.gz
mv nanopb-0.4.9.1-linux-x86 nanopb-0.4.9
@@ -38,7 +33,7 @@ jobs:
- name: Create pull request
uses: peter-evans/create-pull-request@v8
with:
branch: create-pull-request/update-protobufs-${{ github.ref_name }}
branch: create-pull-request/update-protobufs
labels: submodules
title: Update protobufs and classes
commit-message: Update protobufs
+5 -15
View File
@@ -1,8 +1,8 @@
/.pio
/pio
/pio.tar
/web
/web.tar
.pio
pio
pio.tar
web
web.tar
# ignore vscode IDE settings files
.vscode/*
@@ -47,10 +47,6 @@ data/boot/logo.*
managed_components/*
arduino-lib-builder*
dependencies.lock
# JLink / RTT debug artifacts (nRF SoCs)
flash.jlink
rtt_*.txt
idf_component.yml
CMakeLists.txt
/sdkconfig.*
@@ -60,9 +56,3 @@ CMakeLists.txt
.python3
.claude/scheduled_tasks.lock
userPrefs.jsonc.mcp-session-bak
# Fake-NodeDB fixture pipeline (bin/regen-fake-nodedbs.sh)
# JSONL seeds are committed (test/fixtures/nodedb/seed_v25_*.jsonl);
# compiled .proto outputs are ephemeral build artifacts.
build/fixtures/
bin/_generated/
+5 -12
View File
@@ -4,19 +4,19 @@ cli:
plugins:
sources:
- id: trunk
ref: v1.10.0
ref: v1.9.0
uri: https://github.com/trunk-io/plugins
lint:
enabled:
- checkov@3.2.529
- checkov@3.2.528
- renovate@43.150.0
- prettier@3.8.3
- trufflehog@3.95.3
- trufflehog@3.95.2
- yamllint@1.38.0
- bandit@1.9.4
- trivy@0.70.0
- taplo@0.10.0
- ruff@0.15.13
- ruff@0.15.12
- isort@8.0.1
- markdownlint@0.48.0
- oxipng@10.1.1
@@ -26,7 +26,7 @@ lint:
- hadolint@2.14.0
- shfmt@3.6.0
- shellcheck@0.11.0
- black@26.5.1
- black@26.3.1
- git-diff-check
- gitleaks@8.30.1
- clang-format@16.0.3
@@ -34,13 +34,6 @@ lint:
- linters: [ALL]
paths:
- bin/**
# Fake-NodeDB fixture JSONL files contain deterministic synthetic
# public_key_hex (64-char hex) values that gitleaks misidentifies as
# generic-api-key. These are not secrets — they're test fixtures
# produced by bin/gen-fake-nodedb-seed.py with a fixed RNG seed.
- linters: [gitleaks]
paths:
- test/fixtures/nodedb/seed_v25_*.jsonl
runtimes:
enabled:
- python@3.14.4
+4 -4
View File
@@ -1,10 +1,10 @@
{
// See http://go.microsoft.com/fwlink/?LinkId=827846
// for the documentation about the extensions.json format
"recommendations": [
"Jason2866.esp-decoder",
"pioarduino.pioarduino-ide"
"platformio.platformio-ide"
],
"unwantedRecommendations": [
"ms-vscode.cpptools-extension-pack",
"platformio.platformio-ide"
"ms-vscode.cpptools-extension-pack"
]
}
-41
View File
@@ -10,47 +10,6 @@
"isDefault": true
},
"label": "PlatformIO: Build"
},
{
"label": "FleetSuite",
"detail": "Build the SPA if needed and open the FleetSuite desktop window",
"type": "shell",
"command": "./scripts/fleetsuite.sh",
"options": { "cwd": "${workspaceFolder}/mcp-server" },
"problemMatcher": [],
"presentation": {
"reveal": "always",
"panel": "dedicated",
"clear": true
},
"group": "none"
},
{
"label": "FleetSuite: Dev (HMR)",
"detail": "Run the backend + Vite dev server with hot-reload",
"type": "shell",
"command": "./scripts/fleetsuite.sh --dev",
"options": { "cwd": "${workspaceFolder}/mcp-server" },
"problemMatcher": [],
"presentation": {
"reveal": "always",
"panel": "dedicated",
"clear": true
},
"isBackground": true
},
{
"label": "FleetSuite: Rebuild + Run",
"detail": "Force a fresh SPA build, then open the window",
"type": "shell",
"command": "./scripts/fleetsuite.sh --rebuild",
"options": { "cwd": "${workspaceFolder}/mcp-server" },
"problemMatcher": [],
"presentation": {
"reveal": "always",
"panel": "dedicated",
"clear": true
}
}
]
}
+14 -14
View File
@@ -10,18 +10,18 @@ This file (`AGENTS.md`) is a short pointer + quick reference for agents that don
## Quick command reference
| Action | Command |
| -------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Build a firmware variant | `pio run -e <env>` (e.g. `pio run -e rak4631`, `pio run -e heltec-v3`) |
| Build native macOS host binary | `pio run -e native-macos` (Homebrew prereqs + CH341 LoRa setup in `variants/native/portduino/platformio.ini`) |
| Clean + rebuild | `pio run -e <env> -t clean && pio run -e <env>` |
| Flash a device | `pio run -e <env> -t upload --upload-port <port>` (or use the `pio_flash` MCP tool) |
| Run firmware unit tests (native) | `~/.platformio/penv/bin/python -m platformio test -e native > /tmp/test_out.txt 2>&1` then `grep -E 'error:\|PASS\|FAIL\|succeeded\|failed' /tmp/test_out.txt` (redirect first — piping causes line-buffering) |
| Run MCP hardware tests | `./mcp-server/run-tests.sh` |
| Live TUI test runner | `mcp-server/.venv/bin/meshtastic-mcp-test-tui` |
| Format before commit | `trunk fmt` |
| Regenerate protobuf bindings | `bin/regen-protos.sh` |
| Generate CI matrix | `./bin/generate_ci_matrix.py all [--level pr]` |
| Action | Command |
| -------------------------------- | ------------------------------------------------------------------------------------------------------------- |
| Build a firmware variant | `pio run -e <env>` (e.g. `pio run -e rak4631`, `pio run -e heltec-v3`) |
| Build native macOS host binary | `pio run -e native-macos` (Homebrew prereqs + CH341 LoRa setup in `variants/native/portduino/platformio.ini`) |
| Clean + rebuild | `pio run -e <env> -t clean && pio run -e <env>` |
| Flash a device | `pio run -e <env> -t upload --upload-port <port>` (or use the `pio_flash` MCP tool) |
| Run firmware unit tests (native) | `pio test -e native` |
| Run MCP hardware tests | `./mcp-server/run-tests.sh` |
| Live TUI test runner | `mcp-server/.venv/bin/meshtastic-mcp-test-tui` |
| Format before commit | `trunk fmt` |
| Regenerate protobuf bindings | `bin/regen-protos.sh` |
| Generate CI matrix | `./bin/generate_ci_matrix.py all [--level pr]` |
## MCP server (device + test automation)
@@ -64,7 +64,7 @@ Key rotation to never trigger casually: only the **full** factory reset (`factor
- **One MCP call per serial port at a time.** The port lock is exclusive; concurrent calls deadlock. Sequence: open → read/mutate → close, then next device.
- **`userPrefs.jsonc` is session state during tests.** The `_session_userprefs` fixture snapshots + restores it; never edit it from inside a test.
- **Don't speculate about firmware root causes.** When evidence doesn't support a classification, say "unknown" and list what would disambiguate.
- **Run `trunk fmt` before proposing a commit.** The `trunk_check` CI gate will reject unformatted code. Claude Code runs it automatically via the PostToolUse hook in `.claude/settings.json`; trunk's launcher needs `curl` or `wget` to bootstrap its pinned CLI — see **Formatting & the trunk toolchain** in `.github/copilot-instructions.md` for the no-curl bootstrap procedure.
- **Run `trunk fmt` before proposing a commit.** The `trunk_check` CI gate will reject unformatted code.
- **`confirm=True` on destructive MCP tools is a real gate, not a formality.** Don't bypass it via auto-approve settings.
- **Keep code comments minimal — one or two lines, max.** Comment only when the _why_ isn't obvious from the code; never restate what the next line does. No multi-paragraph block comments explaining straightforward changes. The diff and commit message carry the rationale; the code carries the behavior.
- **Use `Throttle` for time-based rate limiting, not raw `millis()` math.** `src/mesh/Throttle.h` provides `Throttle::isWithinTimespanMs(lastMs, intervalMs)` (returns true while inside the cooldown) and `Throttle::execute(&lastMs, intervalMs, func)` (function-pointer form that updates the timestamp on fire). Use these for any "did N ms pass since X" check — raw `millis() > lastMs + N` is rollover-unsafe (breaks after ~49.7 days) and inconsistent with the rest of the codebase. The helpers compute `now - lastMs` with unsigned subtraction, which wraps correctly.
@@ -108,7 +108,7 @@ Sequence these; don't parallelize on the same port.
| `src/modules/` | Feature modules; `Telemetry/Sensor/` has 50+ I2C sensor drivers |
| `variants/` | 200+ hardware variant definitions (`variant.h` + `platformio.ini` per board) |
| `protobufs/` | `.proto` definitions; regenerate with `bin/regen-protos.sh` |
| `test/` | Firmware unit tests (17 suites; `pio test -e native`) |
| `test/` | Firmware unit tests (12 suites; `pio test -e native`) |
| `mcp-server/` | Python MCP server + pytest hardware integration tests |
| `mcp-server/tests/` | Tiered pytest suite: `unit/`, `mesh/`, `telemetry/`, `monitor/`, `recovery/`, `ui/`, `fleet/`, `admin/`, `provisioning/` |
| `.claude/commands/` | Claude Code slash command bodies |
+2 -2
View File
@@ -14,7 +14,7 @@ ENV PIP_BREAK_SYSTEM_PACKAGES=1
RUN apt-get update && apt-get install --no-install-recommends -y \
curl wget g++ zip git ca-certificates pkg-config \
python3-pip python3-grpc-tools \
libgpiod-dev libyaml-cpp-dev libjsoncpp-dev libbluetooth-dev libi2c-dev libuv1-dev \
libgpiod-dev libyaml-cpp-dev libbluetooth-dev libi2c-dev libuv1-dev \
libusb-1.0-0-dev libulfius-dev liborcania-dev libssl-dev \
libx11-dev libinput-dev libxkbcommon-x11-dev libsqlite3-dev libsdl2-dev \
&& apt-get clean && rm -rf /var/lib/apt/lists/* \
@@ -53,7 +53,7 @@ ENV TZ=Etc/UTC
USER root
RUN apt-get update && apt-get --no-install-recommends -y install \
libc-bin libc6 libgpiod3 libyaml-cpp0.8 libjsoncpp26 libi2c0 libuv1t64 libusb-1.0-0-dev \
libc-bin libc6 libgpiod3 libyaml-cpp0.8 libi2c0 libuv1t64 libusb-1.0-0-dev \
liborcania2.3 libulfius2.7t64 libssl3t64 \
libx11-6 libinput10 libxkbcommon-x11-0 libsdl2-2.0-0 \
&& apt-get clean && rm -rf /var/lib/apt/lists/* \
+1 -1
View File
@@ -7,7 +7,7 @@ ENV PIP_ROOT_USER_ACTION=ignore
# hadolint ignore=DL3008
RUN apt-get update && apt-get install --no-install-recommends -y \
g++ git ca-certificates pkg-config \
libgpiod-dev libyaml-cpp-dev libjsoncpp-dev libbluetooth-dev libi2c-dev libuv1-dev \
libgpiod-dev libyaml-cpp-dev libbluetooth-dev libi2c-dev libuv1-dev \
libusb-1.0-0-dev libulfius-dev liborcania-dev libssl-dev \
libx11-dev libinput-dev libxkbcommon-x11-dev libsqlite3-dev libsdl2-dev \
&& apt-get clean && rm -rf /var/lib/apt/lists/* \
+2 -2
View File
@@ -16,7 +16,7 @@ ENV PIP_BREAK_SYSTEM_PACKAGES=1
RUN apk --no-cache add \
bash g++ libstdc++-dev linux-headers zip git ca-certificates libbsd-dev \
py3-pip py3-grpcio-tools \
libgpiod-dev yaml-cpp-dev jsoncpp-dev bluez-dev \
libgpiod-dev yaml-cpp-dev bluez-dev \
libusb-dev i2c-tools-dev libuv-dev openssl-dev pkgconf argp-standalone \
libx11-dev libinput-dev libxkbcommon-dev sqlite-dev sdl2-dev \
&& rm -rf /var/cache/apk/* \
@@ -48,7 +48,7 @@ LABEL org.opencontainers.image.title="Meshtastic" \
USER root
RUN apk --no-cache add \
shadow libstdc++ libbsd libgpiod yaml-cpp jsoncpp libusb \
shadow libstdc++ libbsd libgpiod yaml-cpp libusb \
i2c-tools libuv libx11 libinput libxkbcommon sdl2 \
&& rm -rf /var/cache/apk/* \
&& mkdir -p /var/lib/meshtasticd \
-64
View File
@@ -1,64 +0,0 @@
#!/usr/bin/env python3
"""Post-process protoc-generated Python files to live under a local namespace.
Called by bin/regen-py-protos.sh. Walks the generated *_pb2.py files in the
target directory and rewrites every `meshtastic` reference (imports, dotted
attribute access) to use the new namespace (e.g., `meshtastic_v25`).
Why: the .proto files declare `package meshtastic;`, so protoc emits
`from meshtastic import mesh_pb2 as ...` lines. That would shadow the PyPI
`meshtastic` package which other parts of the mcp-server depend on. Renaming
to a local namespace keeps both available.
Usage:
_rewrite_proto_namespace.py <generated_dir> <new_namespace>
"""
from __future__ import annotations
import pathlib
import re
import sys
def rewrite(dir_path: pathlib.Path, new_ns: str) -> int:
# Standard protoc import forms:
# from meshtastic.X_pb2 import ... (rare, for direct symbol pulls)
# from meshtastic import X_pb2 as ... (common, the cross-file ref)
# import meshtastic.X_pb2 (also possible)
pattern_dotted_from = re.compile(r"^from meshtastic\.", re.MULTILINE)
pattern_bare_from = re.compile(r"^from meshtastic import ", re.MULTILINE)
pattern_dotted_import = re.compile(r"^import meshtastic\.", re.MULTILINE)
count = 0
for p in dir_path.glob("*.py"):
text = p.read_text(encoding="utf-8")
new = pattern_dotted_from.sub(f"from {new_ns}.", text)
new = pattern_bare_from.sub(f"from {new_ns} import ", new)
new = pattern_dotted_import.sub(f"import {new_ns}.", new)
# NOTE: we deliberately leave `meshtastic/X.proto` source-filename
# references inside descriptor strings alone. The descriptor pool is
# keyed by source filename (independent of Python package layout), so
# those don't collide with the PyPI package's descriptors.
if new != text:
p.write_text(new, encoding="utf-8")
count += 1
return count
def main(argv: list[str]) -> int:
if len(argv) != 2:
print("usage: _rewrite_proto_namespace.py <generated_dir> <new_namespace>", file=sys.stderr)
return 2
dir_path = pathlib.Path(argv[0])
new_ns = argv[1]
if not dir_path.is_dir():
print(f"directory not found: {dir_path}", file=sys.stderr)
return 2
n = rewrite(dir_path, new_ns)
print(f"rewrote {n} file(s) in {dir_path} → namespace {new_ns}", file=sys.stderr)
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
+1 -1
View File
@@ -38,4 +38,4 @@ cp bin/device-install.* $OUTDIR/
cp bin/device-update.* $OUTDIR/
echo "Copying manifest"
cp $BUILDDIR/$basename.mt.json $OUTDIR/$basename.mt.json
cp $BUILDDIR/$basename.mt.json $OUTDIR/$basename.mt.json || true
-53
View File
@@ -1,53 +0,0 @@
#!/usr/bin/env python3
"""Collect firmware binary sizes from manifest (.mt.json) files into a single report."""
import json
import os
import sys
def collect_sizes(manifest_dir):
"""Scan manifest_dir for .mt.json files and return {board: size_bytes} dict."""
sizes = {}
for fname in sorted(os.listdir(manifest_dir)):
if not fname.endswith(".mt.json"):
continue
path = os.path.join(manifest_dir, fname)
with open(path) as f:
data = json.load(f)
board = data.get("platformioTarget", fname.replace(".mt.json", ""))
# Find the main firmware .bin size (largest .bin, excluding OTA/littlefs/bleota)
bin_size = None
for entry in data.get("files", []):
name = entry.get("name", "")
if name.startswith("firmware-") and name.endswith(".bin"):
bin_size = entry["bytes"]
break
# Fallback: any .bin that isn't ota/littlefs/bleota
if bin_size is None:
for entry in data.get("files", []):
name = entry.get("name", "")
if name.endswith(".bin") and not any(
x in name for x in ["littlefs", "bleota", "ota"]
):
bin_size = entry["bytes"]
break
if bin_size is not None:
sizes[board] = bin_size
return sizes
if __name__ == "__main__":
if len(sys.argv) != 3:
print(f"Usage: {sys.argv[0]} <manifest_dir> <output.json>", file=sys.stderr)
sys.exit(1)
manifest_dir = sys.argv[1]
output_path = sys.argv[2]
sizes = collect_sizes(manifest_dir)
with open(output_path, "w") as f:
json.dump(sizes, f, indent=2, sort_keys=True)
print(f"Collected sizes for {len(sizes)} targets -> {output_path}")
@@ -1,30 +0,0 @@
---
Lora:
## Ebyte E80-900M22S
## This is a bit experimental
##
##
Module: lr1121
gpiochip: 1 # subtract 32 from the gpio numbers
DIO3_TCXO_VOLTAGE: 1.8
CS: 16 #pin6 / GPIO48 1C0
IRQ: 23 #pin17 / GPIO55 1C7
Busy: 22 #pin16 / GPIO54 1C6
Reset: 25 #pin13 / GPIO57 1D1
spidev: spidev0.0 #pins are (CS=16, CLK=17, MOSI=18, MISO=19)
spiSpeed: 2000000
rfswitch_table:
pins: [DIO5, DIO6, DIO7]
MODE_STBY: [LOW, LOW, LOW]
MODE_RX: [LOW, HIGH, LOW]
MODE_TX: [HIGH, HIGH, LOW]
MODE_TX_HP: [HIGH, LOW, LOW]
MODE_TX_HF: [LOW, LOW, LOW]
MODE_GNSS: [LOW, LOW, HIGH]
MODE_WIFI: [LOW, LOW, LOW]
General:
MACAddressSource: eth0
@@ -1,46 +0,0 @@
---
Lora:
## Ebyte E80-900M22S
## This is a bit experimental
##
##
Module: lr1121
gpiochip: 1 # subtract 32 from the gpio numbers
DIO3_TCXO_VOLTAGE: 1.8
CS: 16 #pin6 / GPIO48 1C0
IRQ: 23 #pin17 / GPIO55 1C7
Busy: 22 #pin16 / GPIO54 1C6
Reset: 25 #pin13 / GPIO57 1D1
spidev: spidev0.0 #pins are (CS=16, CLK=17, MOSI=18, MISO=19)
spiSpeed: 2000000
rfswitch_table:
pins:
- DIO5
- DIO6
MODE_STBY:
- LOW
- LOW
MODE_RX:
- HIGH
- LOW
MODE_TX:
- HIGH
- HIGH
MODE_TX_HP:
- LOW
- HIGH
MODE_TX_HF:
- LOW
- LOW
MODE_GNSS:
- LOW
- LOW
MODE_WIFI:
- LOW
- LOW
General:
MACAddressSource: eth0
@@ -1,30 +0,0 @@
---
Lora:
## Ebyte E80-900M22S
## This is a bit experimental
##
##
Module: lr1121
gpiochip: 1 # subtract 32 from the gpio numbers
DIO3_TCXO_VOLTAGE: 1.8
CS: 16 #pin6 / GPIO48 1C0
IRQ: 23 #pin17 / GPIO55 1C7
Busy: 22 #pin16 / GPIO54 1C6
Reset: 25 #pin13 / GPIO57 1D1
spidev: spidev0.0 #pins are (CS=16, CLK=17, MOSI=18, MISO=19)
spiSpeed: 2000000
rfswitch_table:
pins: [DIO5, DIO6, DIO7]
MODE_STBY: [LOW, LOW, LOW]
MODE_RX: [LOW, LOW, LOW]
MODE_TX: [LOW, HIGH, LOW]
MODE_TX_HP: [HIGH, LOW, LOW]
# MODE_TX_HF: []
# MODE_GNSS: []
MODE_WIFI: [LOW, LOW, LOW]
General:
MACAddressSource: eth0
+1 -5
View File
@@ -5,9 +5,7 @@ Meta:
- raspberry-pi
Lora:
### RAK13300 in Slot 2
Module: sx1262
### RAK13300 in Slot 2 pins
IRQ: 18 #IO6
Reset: 24 # IO4
Busy: 19 # IO5
@@ -15,7 +13,5 @@ Lora:
Enable_Pins:
- 26
- 23
DIO3_TCXO_VOLTAGE: true
DIO2_AS_RF_SWITCH: true
spidev: spidev0.1
# CS: 7
+2 -6
View File
@@ -5,18 +5,14 @@ Meta:
- raspberry-pi
Lora:
### RAK13302 in Slot 2
Module: sx1262
### RAK13302 in Slot 2 pins
IRQ: 18 #IO6
Reset: 24 # IO4
Busy: 19 # IO5
# Ant_sw: 23 # IO3
# Ant_sw: 23 # IO3
Enable_Pins:
- 26
- 23
DIO3_TCXO_VOLTAGE: true
DIO2_AS_RF_SWITCH: true
spidev: spidev0.1
# CS: 7
TX_GAIN_LORA: [9, 9, 10, 11, 9, 8, 9, 10, 10, 10, 11, 12, 12, 12, 12, 12, 12, 12, 12, 10, 9, 8]
-18
View File
@@ -1,18 +0,0 @@
# Station G3 motherboard with a Raspberry Pi Zero 2W as the MCU daughterboard.
# Verify spidev / I2C device paths for your OS — they may differ.
Meta:
name: Station G3
support: community
compatible:
- raspberry-pi
Lora:
Module: sx1262
IRQ: 22 # BCM pin — wiki spec
Reset: 16 # BCM pin — wiki spec
Busy: 24 # BCM pin — wiki spec
# CS: 8 # BCM 8 = SPI0 CE0 (default); uncomment only to override
DIO2_AS_RF_SWITCH: true
DIO3_TCXO_VOLTAGE: true
spidev: spidev0.0
# SX126X_MAX_POWER: 19 # matches Station G2 firmware cap; raise carefully per PA jumper mode
-439
View File
@@ -1,439 +0,0 @@
#!/usr/bin/env python3
"""Deterministic seed-data generator for the fake NodeDB fixture pipeline.
Writes a JSONL file describing N fake-but-realistic Meshtastic peers.
The output is hand-editable and committed; a sibling compile step
(bin/seed-json-to-proto.py) turns it into a binary `meshtastic_NodeDatabase`
v25 protobuf with fresh "now-relative" timestamps.
Determinism contract:
Same --seed -> byte-identical JSONL output, regardless of wall clock.
All timestamps are stored as `*_offset_sec` (seconds before "now"); the
compile step resolves them to absolute epochs at compile time.
Structural fields covered:
* NodeInfoLite header: num, long_name, short_name, hw_model, role,
public_key, snr, channel, hops_away, next_hop, bitfield flags
* PositionLite: lat/long Gaussian around --centroid, altitude, source
* DeviceMetrics: battery/voltage/util/uptime
* EnvironmentMetrics: temp/humidity/pressure/iaq
* StatusMessage: error_code (usually zero)
Active-board allow-list:
hw_model values are restricted to the intersection of
(a) variants with `custom_meshtastic_support_level = 1` in
variants/*/*/platformio.ini, AND
(b) values present in the `HardwareModel` enum in mesh.proto.
See HW_MODEL_WEIGHTS below. Deprecated boards (legacy TLORA / Heltec V1-2 /
classic TBEAM / TBEAM_V0P7 / Nano G1 / etc.) and fuzzer-only sentinels
(PORTDUINO, ANDROID_SIM, DIY_V1, ...) are excluded.
Active-role allow-list:
Excludes ROUTER_CLIENT (deprecated v2.3.15) and REPEATER (deprecated v2.7.11).
"""
from __future__ import annotations
import argparse
import datetime as _dt
import json
import math
import pathlib
import random
import sys
# --------------------------------------------------------------------------
# Active-board allow-list (intersection of tier-1 variants + HardwareModel enum).
# Refresh by running:
# for f in $(find variants -name 'platformio.ini' | xargs grep -lE 'custom_meshtastic_support_level = 1'); do
# grep custom_meshtastic_hw_model_slug $f | awk -F= '{print $2}' | tr -d ' ';
# done | sort -u | comm -12 - <(python3 -c "from meshtastic.protobuf.mesh_pb2 import HardwareModel; print('\\n'.join(HardwareModel.keys()))" | sort)
# --------------------------------------------------------------------------
HW_MODEL_WEIGHTS: dict[str, float] = {
"HELTEC_V3": 14.0,
"T_DECK": 9.0,
"HELTEC_V4": 8.0,
"RAK4631": 8.0,
"HELTEC_MESH_POCKET": 6.0,
"TRACKER_T1000_E": 5.0,
"HELTEC_MESH_NODE_T114": 5.0,
"T_DECK_PRO": 5.0,
"LILYGO_TBEAM_S3_CORE": 4.0,
"HELTEC_WIRELESS_PAPER": 4.0,
"HELTEC_WSL_V3": 3.0,
"T_ECHO": 3.0,
"HELTEC_WIRELESS_TRACKER": 3.0,
"HELTEC_WIRELESS_TRACKER_V2": 2.0,
"HELTEC_VISION_MASTER_E290": 2.0,
"HELTEC_MESH_SOLAR": 2.0,
"SEEED_WIO_TRACKER_L1": 2.0,
"T_LORA_PAGER": 1.5,
"HELTEC_VISION_MASTER_E213": 1.5,
"T_ECHO_PLUS": 1.0,
"MUZI_BASE": 1.0,
"WISMESH_TAP_V2": 1.0,
"THINKNODE_M2": 1.0,
"THINKNODE_M5": 1.0,
"TLORA_T3_S3": 1.0,
# Long tail (uniform low weight across remaining tier-1 boards):
"HELTEC_V4_R8": 0.3,
"HELTEC_VISION_MASTER_T190": 0.3,
"HELTEC_HT62": 0.3,
"HELTEC_MESH_NODE_T096": 0.3,
"M5STACK_C6L": 0.3,
"MINI_EPAPER_S3": 0.3,
"MUZI_R1_NEO": 0.3,
"NOMADSTAR_METEOR_PRO": 0.3,
"RAK3312": 0.3,
"RAK3401": 0.3,
"SEEED_SOLAR_NODE": 0.3,
"SEEED_WIO_TRACKER_L1_EINK": 0.3,
"SENSECAP_INDICATOR": 0.3,
"TBEAM_1_WATT": 0.3,
"THINKNODE_M1": 0.3,
"THINKNODE_M3": 0.3,
"THINKNODE_M6": 0.3,
"T_ECHO_LITE": 0.3,
"WISMESH_TAG": 0.3,
"WISMESH_TAP": 0.3,
"XIAO_NRF52_KIT": 0.3,
"CROWPANEL": 0.3,
}
# Non-deprecated roles only.
ROLE_WEIGHTS: dict[str, float] = {
"CLIENT": 75.0,
"CLIENT_MUTE": 5.0,
"ROUTER": 7.0,
"TRACKER": 3.0,
"SENSOR": 2.0,
"CLIENT_HIDDEN": 2.0,
"ROUTER_LATE": 2.0,
"CLIENT_BASE": 2.0,
"TAK": 1.0,
"TAK_TRACKER": 0.5,
"LOST_AND_FOUND": 0.5,
}
# Name pools — 60 firsts × 60 lasts = 3600 combinations.
FIRSTS = [
"Quick", "Brave", "Silent", "Wild", "Lone", "Bright", "Red", "Blue",
"Green", "Black", "White", "Iron", "Steel", "Copper", "Silver", "Gold",
"Stone", "River", "Forest", "Mountain", "Canyon", "Desert", "Storm", "Sky",
"Solar", "Lunar", "Dawn", "Dusk", "Misty", "Frosty", "Sunny", "Shady",
"Happy", "Sleepy", "Drowsy", "Sneaky", "Sharp", "Smooth", "Rough", "Loud",
"Soft", "Slow", "Fast", "Tall", "Short", "Old", "New", "Tiny",
"Giant", "Hidden", "Lost", "Found", "Wandering", "Roving", "Drifting", "Floating",
"Burning", "Frozen", "Whispering", "Howling",
]
LASTS = [
"Phoenix", "Lion", "Bear", "Wolf", "Hawk", "Eagle", "Fox", "Lynx",
"Cougar", "Coyote", "Raven", "Owl", "Crow", "Falcon", "Heron", "Crane",
"Otter", "Badger", "Bison", "Elk", "Moose", "Stag", "Doe", "Hare",
"Marmot", "Mole", "Beaver", "Squirrel", "Mustang", "Bronco", "Pony", "Colt",
"Cobra", "Viper", "Mamba", "Adder", "Gecko", "Iguana", "Tortoise", "Turtle",
"Salmon", "Trout", "Bass", "Pike", "Shark", "Whale", "Dolphin", "Seal",
"Cactus", "Yucca", "Sage", "Juniper", "Pine", "Cedar", "Aspen", "Oak",
"Bluff", "Mesa", "Arroyo", "Ridge",
]
# Brief callsign pool for licensed-looking suffixes.
CALLSIGN_PREFIXES = ["KX", "WD", "N5", "KE", "AB", "W5", "K1", "KQ", "AE", "NM"]
# Only emojis that fit in 4 UTF-8 bytes (no variation selectors). short_name's
# nanopb max_size:5 (incl. NUL) limits content to 4 bytes. ❄️ / ☀️ would be
# 6 bytes due to U+FE0F variation selector — explicitly excluded.
EMOJI_SHORTNAMES = ["🦊", "🐺", "🦅", "🐢", "🌵", "🔥", "🌙",
"🌊", "🗻", "🌲", "🦌", "🐝", "🦂", "🦉",
"🦇", "🦋"]
# --------------------------------------------------------------------------
# Helpers
# --------------------------------------------------------------------------
NUM_RESERVED = 4 # firmware reserves 0..3 (per NodeDB constants)
NUM_MAX_EXCLUSIVE = 0x80000000 # restrict to positive int32 range for readability
def _weighted_choice(rng: random.Random, weights: dict[str, float]) -> str:
"""Deterministic weighted pick. Uses sorted keys so dict order is fixed."""
keys = sorted(weights.keys())
totals = [weights[k] for k in keys]
return rng.choices(keys, weights=totals, k=1)[0]
def _gen_long_name(rng: random.Random, is_licensed: bool) -> str:
base = f"{rng.choice(FIRSTS)} {rng.choice(LASTS)}"
if is_licensed:
prefix = rng.choice(CALLSIGN_PREFIXES)
# Two trailing alpha chars after the digit; keep within 25 - len(base) - 1
suffix = f" {prefix}{rng.randint(0,9)}{rng.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ')}{rng.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ')}"
# nanopb max_size:25 means C string fits 24 bytes + NUL.
if len(base) + len(suffix) <= 24:
base = base + suffix
# Hard cap to 24 chars (nanopb max_size:25 minus NUL).
return base[:24]
def _gen_short_name(rng: random.Random, long_name: str) -> str:
# 10% emoji-only short_name
if rng.random() < 0.10:
return rng.choice(EMOJI_SHORTNAMES)
first_char = long_name[0].upper() if long_name else "X"
alphanums = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
return first_char + "".join(rng.choices(alphanums, k=3))
def _gen_hops_away(rng: random.Random) -> int:
# Geometric-ish: 0→55%, 1→25%, 2→12%, 3→5%, 4→2%, 5+→1%
r = rng.random()
if r < 0.55:
return 0
if r < 0.80:
return 1
if r < 0.92:
return 2
if r < 0.97:
return 3
if r < 0.99:
return 4
return rng.randint(5, 7)
def _gen_position(
rng: random.Random,
centroid_lat: float,
centroid_lon: float,
spread_km: float,
last_heard_offset_sec: int,
) -> dict:
# 1 deg ≈ 111 km at the equator; we use this as a flat approximation.
lat = centroid_lat + rng.gauss(0.0, spread_km / 111.0)
lon = centroid_lon + rng.gauss(0.0, spread_km / 111.0)
altitude = max(0, round(rng.gauss(1376.0, 250.0))) # T or C valley floor + relief
# Position was reported up to 300s before last_heard.
time_offset_sec = last_heard_offset_sec + rng.randint(0, 300)
return {
"latitude": round(lat, 6),
"longitude": round(lon, 6),
"altitude": altitude,
"time_offset_sec": time_offset_sec,
"location_source": "LOC_INTERNAL",
}
def _gen_telemetry(rng: random.Random) -> dict:
# 5% plugged-in (battery_level == 101); rest uniform [10..100].
if rng.random() < 0.05:
battery_level = 101
voltage = 4.20
else:
battery_level = rng.randint(10, 100)
voltage = round(3.3 + (battery_level / 100.0) * 0.9, 3)
# Beta distributions for low/right-skewed metrics; randomly draw via gammavariate.
def _beta(a: float, b: float) -> float:
x = rng.gammavariate(a, 1.0)
y = rng.gammavariate(b, 1.0)
return x / (x + y)
channel_utilization = round(_beta(2.0, 15.0) * 100.0, 2)
air_util_tx = round(_beta(1.5, 20.0) * 10.0, 3)
uptime_seconds = int(rng.expovariate(1.0 / 86400.0))
return {
"battery_level": battery_level,
"voltage": voltage,
"channel_utilization": channel_utilization,
"air_util_tx": air_util_tx,
"uptime_seconds": uptime_seconds,
}
def _gen_environment(rng: random.Random) -> dict:
return {
"temperature": round(rng.gauss(22.0, 8.0), 2),
"relative_humidity": round(min(100.0, max(0.0, rng.gauss(55.0, 20.0))), 2),
"barometric_pressure": round(rng.gauss(1013.0, 8.0), 2),
"iaq": int(min(500, max(0, round(rng.gauss(50.0, 30.0))))),
}
def _gen_status(rng: random.Random) -> dict:
# `StatusMessage` (mesh.proto:1445) has a single free-form `string status`.
# Most peers report a healthy short status; occasional alert string.
healthy = ["OK", "online", "active", "running", "ready", "nominal"]
alert = ["low-batt", "no-gps", "weak-signal", "rebooted", "offline-soon"]
if rng.random() < 0.92:
return {"status": rng.choice(healthy)}
return {"status": rng.choice(alert)}
def _gen_node(
rng: random.Random,
num: int,
centroid_lat: float,
centroid_lon: float,
spread_km: float,
coverage: dict[str, float],
last_heard_mean_sec: int,
last_heard_max_sec: int,
) -> dict:
is_licensed = rng.random() < 0.05
long_name = _gen_long_name(rng, is_licensed)
short_name = _gen_short_name(rng, long_name)
hw_model = _weighted_choice(rng, HW_MODEL_WEIGHTS)
role = _weighted_choice(rng, ROLE_WEIGHTS)
has_public_key = rng.random() < 0.92
public_key_hex = (
"".join(f"{rng.randint(0,255):02x}" for _ in range(32)) if has_public_key else ""
)
snr = round(max(-20.0, min(12.0, rng.gauss(6.0, 4.0))), 2)
channel = 0 if rng.random() < 0.90 else rng.randint(1, 7)
hops_away = _gen_hops_away(rng)
next_hop = rng.randint(0, 255) if hops_away > 0 else 0
last_heard_offset_sec = int(min(rng.expovariate(1.0 / last_heard_mean_sec), last_heard_max_sec))
bitfield = {
"has_user": True,
"is_favorite": rng.random() < 0.08,
"is_muted": rng.random() < 0.03,
"via_mqtt": rng.random() < 0.12,
"is_ignored": rng.random() < 0.01,
"is_licensed": is_licensed,
"has_is_unmessagable": True,
"is_unmessagable": rng.random() < 0.02,
"is_key_manually_verified": rng.random() < 0.04,
}
node: dict = {
"num": f"0x{num:08x}",
"long_name": long_name,
"short_name": short_name,
"hw_model": hw_model,
"role": role,
"public_key_hex": public_key_hex,
"snr": snr,
"channel": channel,
"hops_away": hops_away,
"next_hop": next_hop,
"last_heard_offset_sec": last_heard_offset_sec,
"bitfield": bitfield,
"position": (
_gen_position(rng, centroid_lat, centroid_lon, spread_km, last_heard_offset_sec)
if rng.random() < coverage["position"]
else None
),
"telemetry": _gen_telemetry(rng) if rng.random() < coverage["telemetry"] else None,
"environment": _gen_environment(rng) if rng.random() < coverage["environment"] else None,
"status": _gen_status(rng) if rng.random() < coverage["status"] else None,
}
return node
def _parse_my_node_num(s: str | None) -> int | None:
if s is None:
return None
s = s.strip()
if s.startswith("0x") or s.startswith("0X"):
return int(s, 16)
return int(s)
def main(argv: list[str]) -> int:
p = argparse.ArgumentParser(
description="Deterministic JSONL seed for the fake NodeDB fixture.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
p.add_argument("--count", type=int, required=True, help="Number of fake nodes to emit.")
p.add_argument("--seed", type=int, required=True, help="Deterministic seed.")
p.add_argument("--out", required=True, help="Output JSONL path.")
p.add_argument(
"--centroid",
default="33.1284,-107.2528",
help="LAT,LON centroid (default: Truth or Consequences, NM).",
)
p.add_argument("--spread-km", type=float, default=60.0, help="Gaussian std-dev in km.")
p.add_argument("--position-coverage", type=float, default=0.85)
p.add_argument("--telemetry-coverage", type=float, default=0.70)
p.add_argument("--environment-coverage", type=float, default=0.25)
p.add_argument("--status-coverage", type=float, default=0.40)
p.add_argument("--my-node-num", default=None, help="Exclude this NodeNum from generated set (hex or dec).")
p.add_argument("--last-heard-mean-sec", type=int, default=3600)
p.add_argument("--last-heard-max-sec", type=int, default=7 * 86400)
args = p.parse_args(argv)
if args.count <= 0:
print("--count must be positive", file=sys.stderr)
return 2
try:
centroid_lat, centroid_lon = (float(s) for s in args.centroid.split(","))
except ValueError:
print(f"--centroid must be LAT,LON; got {args.centroid!r}", file=sys.stderr)
return 2
my_node_num = _parse_my_node_num(args.my_node_num)
rng = random.Random(args.seed)
# 1) Generate a unique deterministic set of NodeNums.
nums: set[int] = set()
while len(nums) < args.count:
n = rng.randrange(NUM_RESERVED, NUM_MAX_EXCLUSIVE)
if my_node_num is not None and n == my_node_num:
continue
nums.add(n)
ordered_nums = sorted(nums) # sort to fix output order independent of set hash
# 2) Per-node generation (in num order, single RNG continues).
coverage = {
"position": args.position_coverage,
"telemetry": args.telemetry_coverage,
"environment": args.environment_coverage,
"status": args.status_coverage,
}
nodes = [
_gen_node(
rng,
n,
centroid_lat,
centroid_lon,
args.spread_km,
coverage,
args.last_heard_mean_sec,
args.last_heard_max_sec,
)
for n in ordered_nums
]
# 3) Write JSONL.
out_path = pathlib.Path(args.out)
out_path.parent.mkdir(parents=True, exist_ok=True)
# `generated_at_iso` is informational; it does NOT affect determinism because
# we derive it from the seed, not from wall clock. (Same seed -> same string.)
generated_at = _dt.datetime.fromtimestamp(args.seed, tz=_dt.timezone.utc).isoformat().replace("+00:00", "Z")
meta = {
"_meta": {
"version": 25,
"seed": args.seed,
"count": args.count,
"centroid": [centroid_lat, centroid_lon],
"spread_km": args.spread_km,
"generated_at_iso": generated_at,
"my_node_num_excluded": (None if my_node_num is None else f"0x{my_node_num:08x}"),
"coverage": coverage,
"last_heard_mean_sec": args.last_heard_mean_sec,
"last_heard_max_sec": args.last_heard_max_sec,
}
}
with out_path.open("w", encoding="utf-8") as f:
# `ensure_ascii=False` so emoji short_names survive. `sort_keys=True` for
# determinism (insertion order varies by Python version otherwise).
f.write(json.dumps(meta, ensure_ascii=False, sort_keys=True) + "\n")
for node in nodes:
f.write(json.dumps(node, ensure_ascii=False, sort_keys=True) + "\n")
print(f"wrote {args.count} nodes to {out_path} ({out_path.stat().st_size} bytes)", file=sys.stderr)
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
+4 -8
View File
@@ -293,12 +293,9 @@ if ("HAS_TFT", 1) in env.get("CPPDEFINES", []):
board_arch = infer_architecture(env.BoardConfig())
should_skip_manifest = board_arch is None
# Most platforms can generate the manifest as part of the default 'buildprog' target.
# Typically this passes success/failure properly.
mtjson_deps = ["buildprog"]
if platform.name == "espressif32":
# On ESP32, we need to explicitly depend upon the binary to prevent fake-success upon failure.
mtjson_deps = ["$BUILD_DIR/${PROGNAME}.bin"]
# For host/native envs, avoid depending on 'buildprog' (some targets don't define it)
mtjson_deps = [] if should_skip_manifest else ["buildprog"]
if not should_skip_manifest and platform.name == "espressif32":
# Build littlefs image as part of mtjson target
# Equivalent to `pio run -t buildfs`
target_lfs = env.DataToBin(
@@ -312,8 +309,7 @@ if should_skip_manifest:
env.AddCustomTarget(
name="mtjson",
# For host/native envs, avoid depending on 'buildprog' (some targets don't define it)
dependencies=[],
dependencies=mtjson_deps,
actions=[skip_manifest],
title="Meshtastic Manifest (skipped)",
description="mtjson generation is skipped for native environments",
-73
View File
@@ -1,73 +0,0 @@
#!/usr/bin/env bash
# Regenerate the fake-NodeDB fixtures: produces 250 / 500 / 1000 / 2000-node
# JSONL seed files + their compiled v25 protobufs.
#
# Layout:
# test/fixtures/nodedb/seed_v25_<N>.jsonl — COMMITTED, hand-editable.
# build/fixtures/nodedb/nodes_v25_<N>.proto — .gitignored, build artifact.
# Drop into /prefs/nodes.proto.
#
# Daily use: ./bin/regen-fake-nodedbs.sh
# - Recompiles protos from committed seeds (fresh wall-clock timestamps).
# Intentional seed bump: REGEN_SEEDS=yes ./bin/regen-fake-nodedbs.sh
# - Overwrites the committed JSONL files with freshly-seeded data.
set -euo pipefail
cd "$(dirname "$0")/.."
# 1) Make sure the Python protobuf bindings exist (in-tree generation; .gitignored).
if [[ ! -d bin/_generated/meshtastic ]]; then
echo "regenerating Python protobuf bindings (one-time)..."
./bin/regen-py-protos.sh
fi
# 2) Pick a Python interpreter that has the meshtastic deps installed.
# Prefer the mcp-server venv (most likely to be set up by the operator).
PY="python3"
for cand in mcp-server/.venv/bin/python3 .venv/bin/python3; do
if [[ -x "$cand" ]]; then
PY="$cand"
break
fi
done
# 3) Pinned seeds per size — bump only when you intentionally want different
# structural data committed. Parallel arrays so the script works on
# macOS bash 3.2 (no `declare -A`).
SIZES=(250 500 1000 2000)
SEEDS=(20260511 20260512 20260513 20260514)
REGEN_SEEDS="${REGEN_SEEDS:-no}"
mkdir -p build/fixtures/nodedb test/fixtures/nodedb
for i in 0 1 2 3; do
n="${SIZES[$i]}"
seed="${SEEDS[$i]}"
jsonl=$(printf "test/fixtures/nodedb/seed_v25_%04d.jsonl" "$n")
proto=$(printf "build/fixtures/nodedb/nodes_v25_%04d.proto" "$n")
if [[ "$REGEN_SEEDS" == "yes" || ! -f "$jsonl" ]]; then
$PY bin/gen-fake-nodedb-seed.py \
--count "$n" \
--seed "$seed" \
--out "$jsonl" \
--centroid 33.1284,-107.2528 \
--spread-km 60 \
--position-coverage 0.85 \
--telemetry-coverage 0.70 \
--environment-coverage 0.25 \
--status-coverage 0.40
echo " seed: $jsonl ($(wc -c < "$jsonl") bytes)"
fi
$PY bin/seed-json-to-proto.py --in "$jsonl" --out "$proto"
echo " proto: $proto ($(wc -c < "$proto") bytes)"
done
echo ""
echo "Done. To load on Portduino native:"
echo " cp build/fixtures/nodedb/nodes_v25_1000.proto ~/.portduino/default/prefs/nodes.proto"
echo ""
echo "To push to a hardware device:"
echo " Use the mcp-server tool: push_fake_nodedb(size=1000, target=\"hardware\", port=\"/dev/cu.usbmodemXXXX\", confirm=True)"
-51
View File
@@ -1,51 +0,0 @@
#!/usr/bin/env bash
# Regenerate Python protobuf bindings from the in-tree `protobufs/` submodule
# into `bin/_generated/`. Called by bin/regen-fake-nodedbs.sh; also useful as
# a standalone refresh after any change to a .proto file.
#
# Output is .gitignored — bindings are a build artifact.
#
# Namespace rewrite:
# The .proto files declare `package meshtastic;`, which makes protoc emit
# imports like `from meshtastic import mesh_pb2`. That conflicts with the
# PyPI `meshtastic` package (which the mcp-server relies on for its
# SerialInterface/BLEInterface transport). We post-process the generated
# files to live under `meshtastic_v25` instead — both the directory layout
# and all internal imports — so they coexist cleanly with the PyPI package.
set -euo pipefail
cd "$(dirname "$0")/.."
if ! command -v protoc >/dev/null 2>&1; then
echo "ERROR: protoc not found in PATH." >&2
echo " macOS: brew install protobuf" >&2
echo " Ubuntu/Debian: apt install protobuf-compiler" >&2
exit 1
fi
OUT=bin/_generated
LOCAL_NS=meshtastic_v25
rm -rf "$OUT"
mkdir -p "$OUT"
# 1) Generate from the in-tree protos. nanopb.proto first so its descriptor
# is available for the [(nanopb).*] options on other messages.
protoc \
--proto_path=protobufs \
--python_out="$OUT" \
protobufs/nanopb.proto \
protobufs/meshtastic/*.proto
# 2) Move the generated `meshtastic/` directory to `meshtastic_v25/`.
mv "$OUT/meshtastic" "$OUT/$LOCAL_NS"
# 3) Rewrite internal imports: any reference to `meshtastic.X_pb2` or
# `from meshtastic import X_pb2` becomes `meshtastic_v25.*`.
python3 bin/_rewrite_proto_namespace.py "$OUT/$LOCAL_NS" "$LOCAL_NS"
# 4) Make the package importable.
touch "$OUT/__init__.py"
touch "$OUT/$LOCAL_NS/__init__.py"
echo "regenerated Python protobuf bindings -> $OUT/$LOCAL_NS/ (namespace: $LOCAL_NS)" >&2
-249
View File
@@ -1,249 +0,0 @@
#!/usr/bin/env bash
# Run native PlatformIO unit tests and emit a single, unambiguous RED/AMBER/GREEN verdict.
#
# Why this exists: PlatformIO reports failures three different ways ([FAILED], :FAIL:,
# [ERRORED]) and an all-pass run prints "N succeeded" with NO "0 failed" clause — so naive
# greps produce false greens (see .notes/test-passfail-filter.md). This script encodes the
# correct logic once, and cross-checks the number of suites that actually ran against the
# canonical set in test/ so a suite silently going missing shows up as AMBER, not green.
#
# Usage:
# ./bin/run-tests.sh # run all suites, full RAG + count cross-check
# ./bin/run-tests.sh -f test_utf8 # run one suite (no count cross-check)
# ./bin/run-tests.sh -e native # override env (default: coverage)
# ./bin/run-tests.sh --quiet # only print the final RESULT line
#
# Exit codes: 0 = GREEN, 1 = RED, 2 = AMBER.
#
# The final line is machine-readable, e.g.:
# RESULT: GREEN 19/19 suites passed
# RESULT: AMBER 17/19 suites ran (missing: test_radio test_serial) — all that ran passed
# RESULT: RED test_traffic_management: 1 failed (or: build/crash error)
# RESULT: RED sanitizer fault — SUMMARY: AddressSanitizer: 1272 byte(s) leaked (tests may have
# all passed; the coverage build aborts at exit on an ASan/LSan fault — often shown only
# as [ERRORED]/SIGHUP. The script names it and points at running the binary bare.)
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
cd "$ROOT_DIR"
ENV="coverage"
FILTER=""
QUIET=false
PASSTHRU=()
while [[ $# -gt 0 ]]; do
case "$1" in
-f)
FILTER="$2"
PASSTHRU+=("-f" "$2")
shift 2
;;
-e)
ENV="$2"
shift 2
;;
--quiet)
QUIET=true
shift
;;
*)
PASSTHRU+=("$1")
shift
;;
esac
done
# Locate pio (PATH, then the standard PlatformIO venv).
PIO="$(command -v pio || command -v platformio || echo "$HOME/.platformio/penv/bin/pio")"
if [[ ! -x $PIO ]] && ! command -v "$PIO" >/dev/null 2>&1; then
echo "RESULT: RED pio not found (looked in PATH and ~/.platformio/penv/bin)"
exit 1
fi
LOG="$(mktemp -t meshtest.XXXXXX.log)"
MARKER=""
PROGRESS_PID=""
trap 'rm -f "$LOG" "${MARKER:-}"; [[ -n ${PROGRESS_PID:-} ]] && kill "$PROGRESS_PID" 2>/dev/null' EXIT
# Canonical suite set = the directories in test/. This is the source of truth for
# "what should run"; a filtered run only expects its filtered suite.
mapfile -t ALL_SUITES < <(find test -maxdepth 1 -type d -name 'test_*' -printf '%f\n' | sort)
EXPECTED_COUNT=${#ALL_SUITES[@]}
# Cached object-count for this env, written after each completed build (in the gitignored build
# dir). Used as the progress denominator: accurate for a full rebuild (every object recompiles),
# only a rough upper bound for an incremental run.
BASELINE_FILE=".pio/build/${ENV}/.runtests-objcount"
# Progress trail file (gitignored build dir). ALWAYS written so a backgrounded/piped run can be
# checked mid-build with `tail -f` — that's the whole point: don't fly blind on a 20-min rebuild.
PROGRESS_FILE=".pio/build/${ENV}/.runtests-progress"
# --- Progress heartbeat ------------------------------------------------------
# Emit ONE status line every few seconds: build = objects (re)compiled this run / cached total +
# best-effort ETA; test = suites finished / expected. Appends to $PROGRESS_FILE always (tail it to
# check on a backgrounded run); also live-updates the tty when $5=1 (interactive --quiet). Never
# touches $LOG, which is parsed for the verdict, so piped/CI captures stay clean.
progress_monitor() {
local marker="$1" objtotal="$2" testtotal="$3" pfile="$4" totty="$5" start now el done ran eta line
start=$(date +%s)
while :; do
now=$(date +%s)
el=$((now - start))
if grep -q 'Testing\.\.\.' "$LOG" 2>/dev/null; then
ran=$(grep -cE "${ENV}:test_[a-z0-9_]+ \[(PASSED|FAILED|ERRORED)\]" "$LOG" 2>/dev/null)
line=$(printf '[test] %s/%s suites done — %dm%02ds' "$ran" "$testtotal" $((el / 60)) $((el % 60)))
else
done=$(find ".pio/build/${ENV}" -name '*.o' -newer "$marker" 2>/dev/null | wc -l)
if ((objtotal > 0 && done > 0)); then
eta=$((objtotal > done ? (objtotal - done) * el / done : 0))
line=$(printf '[build] %d/%d objs — %dm%02ds — ETA ~%dm%02ds' \
"$done" "$objtotal" $((el / 60)) $((el % 60)) $((eta / 60)) $((eta % 60)))
else
# done==0 (incremental: nothing to rebuild yet) or no cached baseline — no ETA yet.
line=$(printf '[build] %d objs compiled — %dm%02ds' "$done" $((el / 60)) $((el % 60)))
fi
fi
printf '%s\n' "$line" >>"$pfile" 2>/dev/null # file trail (always)
[[ $totty == 1 ]] && printf '\r\033[K%s' "$line" >/dev/tty 2>/dev/null # live line (human)
sleep 4
done
}
# Launch the heartbeat for every run. It writes the progress file unconditionally; the live tty
# line only when interactive AND --quiet (where pio's own output is hidden — otherwise pio's
# streamed compile lines already show progress and a \r line would just fight them).
mkdir -p ".pio/build/${ENV}" 2>/dev/null || true
: >"$PROGRESS_FILE" 2>/dev/null || true
MARKER="$(mktemp -t meshtest-mark.XXXXXX)"
TOTTY=0
{ $QUIET && [[ -t 1 ]]; } && TOTTY=1
progress_monitor "$MARKER" "$(cat "$BASELINE_FILE" 2>/dev/null || echo 0)" \
"$([[ -n $FILTER ]] && echo 1 || echo "$EXPECTED_COUNT")" "$PROGRESS_FILE" "$TOTTY" &
PROGRESS_PID=$!
if ! $QUIET; then
echo "Running: $PIO test -e $ENV ${PASSTHRU[*]-} (expecting $EXPECTED_COUNT suites)"
fi
echo "progress: tail -f $PROGRESS_FILE" >&2
# Run pio, tee to log. PIPESTATUS[0] is pio's real exit (NOT tee's).
if $QUIET; then
"$PIO" test -e "$ENV" "${PASSTHRU[@]}" >"$LOG" 2>&1
else
"$PIO" test -e "$ENV" "${PASSTHRU[@]}" 2>&1 | tee "$LOG"
fi
PIO_RC=${PIPESTATUS[0]}
# Stop the heartbeat, clear its line, and cache this build's object total for next time.
if [[ -n $PROGRESS_PID ]]; then
kill "$PROGRESS_PID" 2>/dev/null
wait "$PROGRESS_PID" 2>/dev/null
PROGRESS_PID=""
# Clear the live line only if we were writing one — opening /dev/tty when there is none is
# itself a redirect-open error the trailing 2>/dev/null cannot suppress.
[[ $TOTTY == 1 ]] && printf '\r\033[K' >/dev/tty 2>/dev/null
fi
[[ -d ".pio/build/${ENV}" ]] && find ".pio/build/${ENV}" -name '*.o' 2>/dev/null | wc -l >"$BASELINE_FILE" 2>/dev/null || true
# --- Outcome detection -------------------------------------------------------
# The SAME outcome is spelled differently depending on which layer emitted the line — this is
# the trap that produces false greens (grepping ":PASS" misses pio's "[PASSED]", grepping
# "[FAILED]" misses Unity's ":FAIL:"). So every regex below matches BOTH spellings:
# pass: Unity per-assertion ":PASS" | pio per-suite "[PASSED]" | summary "N succeeded"
# fail: Unity per-assertion ":FAIL:" | pio per-suite "[FAILED]" | summary "M failed"
# error: pio build/crash "[ERRORED]" | Unity "M Failures" | compiler "error:"
# Match \b after :PASS/:FAIL so ":PASSED"/":FAILED" forms are also caught either way.
FAIL_RE=':FAIL\b|\[FAILED\]|\[ERRORED\]|[1-9][0-9]* failed|[0-9]+ Tests [1-9][0-9]* Failures|error:|undefined reference|Segmentation fault|terminate called|SIGHUP|SIGSEGV|SIGABRT'
# Positive proof tests actually ran & passed (absence != success). Accept any pass spelling:
# the per-test/per-suite tokens OR a success summary line.
PASS_RE=':PASS\b|\[PASSED\]|test cases: *[0-9]+ succeeded|[0-9]+ Tests 0 Failures'
# Sanitizer (ASan/LSan/UBSan/TSan) fault signatures. The coverage build is sanitizer-instrumented
# and aborts NON-ZERO at exit on a fault — most often a LeakSanitizer leak — AFTER every test has
# already printed [PASSED]. pio then reports [ERRORED]/SIGHUP with no :FAIL: anywhere, so it
# masquerades as a phantom "N-1 of N succeeded". See .notes/test-passfail-filter.md.
# Match only real FAULT lines, never the benign "AddressSanitizer: failed to intercept '...'"
# startup noise that prints on every sanitizer run (it'd mislabel a normal [FAILED] as a leak).
# Formats per LLVM/Google sanitizer docs: ASan/LSan emit "==PID==ERROR: <San>: ...", UBSan emits
# "file:line:col: runtime error: ...", TSan emits "WARNING: ThreadSanitizer: ..."; all close with
# a "SUMMARY: <San>: ..." line (LSan-under-ASan reports its SUMMARY as "AddressSanitizer").
SAN_RE='(ERROR|WARNING): (Address|Leak|Thread|UndefinedBehavior)Sanitizer:|SUMMARY: (Address|Leak|Thread|UndefinedBehavior)Sanitizer:|Direct leak of|Indirect leak of|detected memory leaks|heap-use-after-free|heap-buffer-overflow|stack-buffer-overflow|attempting double-free|LeakSanitizer has encountered a fatal error|runtime error:'
# Suites that produced a per-suite verdict. pio emits "coverage:test_x [PASSED|FAILED|ERRORED]";
# a SKIPPED suite (hardware-only on native) is "accounted for" too, so it doesn't read as missing.
mapfile -t RAN_SUITES < <(grep -oE "${ENV}:test_[a-z0-9_]+ \[(PASSED|FAILED|ERRORED)\]" "$LOG" |
sed -E "s/^${ENV}:(test_[a-z0-9_]+) .*/\1/" | sort -u)
RAN_COUNT=${#RAN_SUITES[@]}
# Suites pio explicitly skipped (don't count these as "missing" in the canonical cross-check).
mapfile -t SKIPPED_SUITES < <(grep -oE "${ENV}:test_[a-z0-9_]+.*\bSKIPPED\b" "$LOG" |
grep -oE "test_[a-z0-9_]+" | sort -u)
verdict_red() {
local detail bin
detail="$(grep -nE '\[FAILED\]|:FAIL:|\[ERRORED\]' "$LOG" | head -3 | sed 's/^/ /')"
echo ""
echo "RED — failures detected:"
[[ -n $detail ]] && echo "$detail"
grep -E 'test cases:' "$LOG" | tail -1 | sed 's/^/ /'
# Path to the test binary for the "run it bare" hint. For native/coverage the test program is
# the env executable (e.g. .pio/build/coverage/meshtasticd), NOT a file named 'program'.
bin="$(find ".pio/build/${ENV}" -maxdepth 1 -type f -executable ! -name '*.so' 2>/dev/null | head -1)"
[[ -z $bin ]] && bin=".pio/build/${ENV}/<program> (build it first: $PIO test -e ${ENV} ${FILTER:+-f $FILTER} --without-testing)"
# Sanitizer fault (ASan/LSan/UBSan/TSan): name the real cause instead of "build/crash error".
if grep -qE "$SAN_RE" "$LOG"; then
grep -nE "$SAN_RE" "$LOG" | head -4 | sed 's/^/ /'
echo " -> sanitizer fault: if every test above is PASS, this is an exit-time abort, not a failed assertion."
echo " -> read the full report by running the binary BARE (gdb hides it via ptrace): ./$bin 2>&1 | tail -40"
echo "RESULT: RED sanitizer fault — $(grep -ohE 'SUMMARY: [A-Za-z]+Sanitizer:.*' "$LOG" | tail -1 || echo 'see report above')"
exit 1
fi
# All tests passed but the process still aborted at EXIT (ERRORED/SIGHUP/SIGABRT) and the
# sanitizer report was swallowed by the runner (often surfaced only as SIGHUP). Almost always a
# sanitizer fault — point at how to surface it rather than calling it a generic crash.
if grep -qE "$PASS_RE" "$LOG" && grep -qE '\[ERRORED\]|SIGHUP|SIGABRT' "$LOG" && ! grep -qE ':FAIL\b|\[FAILED\]' "$LOG"; then
echo " -> all tests passed but the process aborted at EXIT — likely an ASan/LSan fault whose report"
echo " the runner swallowed (commonly shown as SIGHUP). Run the binary BARE to see it: ./$bin 2>&1 | tail -40"
echo "RESULT: RED exit-time abort (tests passed; likely sanitizer — see hint above)"
exit 1
fi
echo "RESULT: RED $(grep -oE '[0-9]+ failed' "$LOG" | tail -1 || echo 'build/crash error')"
exit 1
}
# RED: pio non-zero, any failure marker, or no positive summary at all (build died early).
if [[ $PIO_RC -ne 0 ]] || grep -qE "$FAIL_RE" "$LOG"; then
verdict_red
fi
if ! grep -qE "$PASS_RE" "$LOG"; then
echo ""
echo "RESULT: RED no success summary found (build error / no tests ran?) — see log"
exit 1
fi
# AMBER: everything that ran passed, but (full run only) a canonical suite neither ran NOR was
# explicitly skipped — i.e. it silently went missing. SKIPPED suites are accounted for.
ACCOUNTED_COUNT=$((RAN_COUNT + ${#SKIPPED_SUITES[@]}))
if [[ -z $FILTER && $ACCOUNTED_COUNT -lt $EXPECTED_COUNT ]]; then
missing=()
for s in "${ALL_SUITES[@]}"; do
printf '%s\n' "${RAN_SUITES[@]}" "${SKIPPED_SUITES[@]}" | grep -qx "$s" || missing+=("$s")
done
echo ""
echo "RESULT: AMBER ${RAN_COUNT}/${EXPECTED_COUNT} suites ran (missing: ${missing[*]}) — all that ran passed"
exit 2
fi
# GREEN.
if [[ -n $FILTER ]]; then
echo "RESULT: GREEN ${RAN_COUNT} suite(s) passed (filtered: $FILTER)"
else
echo "RESULT: GREEN ${RAN_COUNT}/${EXPECTED_COUNT} suites passed"
fi
exit 0
-342
View File
@@ -1,342 +0,0 @@
#!/usr/bin/env python3
"""Compile a committed seed JSONL into a binary meshtastic_NodeDatabase v25 proto.
The input is produced by `bin/gen-fake-nodedb-seed.py`. Timestamps in the JSONL
are stored as `*_offset_sec` (seconds before "now"); this script resolves them
to absolute epochs using `--now-epoch` (default: current wall clock).
Output is a raw `pb_encode`-compatible binary that can be dropped at
`/prefs/nodes.proto` on the device (Portduino prefs dir or hardware via
XModem) and loaded by `NodeDB::loadFromDisk` at boot.
Wire format reference:
protobufs/meshtastic/deviceonly.proto (NodeDatabase, NodeInfoLite, sat entries)
src/mesh/NodeDB.h:467-484 (bitfield bit positions)
src/mesh/NodeDB.cpp:1523-1524 (pb_decode entry point)
"""
from __future__ import annotations
import argparse
import json
import pathlib
import sys
import time
from typing import Any
# Prefer the in-tree generated Python protobuf bindings (bin/_generated/meshtastic_v25/)
# because the firmware branch's protos (v25 NodeDatabase satellite arrays, slim
# NodeInfoLite) are typically newer than what the PyPI `meshtastic` package
# ships. Run `bin/regen-py-protos.sh` to (re)generate.
#
# Namespace note: the local bindings live under `meshtastic_v25` (NOT `meshtastic`)
# to avoid shadowing the PyPI `meshtastic` package — bin/regen-py-protos.sh
# post-processes the protoc output to rename the package.
_HERE = pathlib.Path(__file__).resolve().parent
_LOCAL_PROTO_DIR = _HERE / "_generated"
if _LOCAL_PROTO_DIR.is_dir():
sys.path.insert(0, str(_LOCAL_PROTO_DIR))
try:
from meshtastic_v25.deviceonly_pb2 import ( # type: ignore[import-not-found]
NodeDatabase,
NodeInfoLite,
NodePositionEntry,
NodeTelemetryEntry,
NodeEnvironmentEntry,
NodeStatusEntry,
PositionLite,
)
from meshtastic_v25.mesh_pb2 import HardwareModel, Position, StatusMessage # type: ignore[import-not-found]
from meshtastic_v25.config_pb2 import Config # type: ignore[import-not-found]
from meshtastic_v25.telemetry_pb2 import DeviceMetrics, EnvironmentMetrics # type: ignore[import-not-found]
except ImportError as local_err:
# Fall back to the PyPI package if in-tree bindings haven't been generated.
# Will fail the v25 assertion below if the PyPI package predates the
# satellite-DB schema, but at least gives a clear "run regen-py-protos.sh"
# error message instead of an opaque ImportError.
try:
from meshtastic.protobuf.deviceonly_pb2 import (
NodeDatabase,
NodeInfoLite,
NodePositionEntry,
NodeTelemetryEntry,
NodeEnvironmentEntry,
NodeStatusEntry,
PositionLite,
)
from meshtastic.protobuf.mesh_pb2 import HardwareModel, Position, StatusMessage
from meshtastic.protobuf.config_pb2 import Config
from meshtastic.protobuf.telemetry_pb2 import DeviceMetrics, EnvironmentMetrics
except ImportError as pypi_err:
print(
"ERROR: could not import meshtastic protobuf bindings.\n"
" In-tree generation: run `bin/regen-py-protos.sh` (requires protoc).\n"
" PyPI fallback: `pip install meshtastic` (may lag firmware branch).\n"
f" local error (meshtastic_v25): {local_err}\n"
f" pypi error (meshtastic.protobuf): {pypi_err}",
file=sys.stderr,
)
sys.exit(1)
# Fail loudly if bindings predate v25 (no satellite arrays).
assert (
hasattr(NodeDatabase, "DESCRIPTOR")
and "positions" in NodeDatabase.DESCRIPTOR.fields_by_name
), (
"Loaded meshtastic bindings are older than v25 (NodeDatabase.positions missing). "
"Run `bin/regen-py-protos.sh` against the in-tree protobufs/ submodule."
)
# ---------------------------------------------------------------------------
# Bitfield bit positions (mirror src/mesh/NodeDB.h:467-484).
# ---------------------------------------------------------------------------
BIT_IS_KEY_MANUALLY_VERIFIED = 0
BIT_IS_MUTED = 1
BIT_VIA_MQTT = 2
BIT_IS_FAVORITE = 3
BIT_IS_IGNORED = 4
BIT_HAS_USER = 5
BIT_IS_LICENSED = 6
BIT_IS_UNMESSAGABLE = 7
BIT_HAS_IS_UNMESSAGABLE = 8
BITFIELD_LAYOUT = (
# JSON key bit position
("is_key_manually_verified", BIT_IS_KEY_MANUALLY_VERIFIED),
("is_muted", BIT_IS_MUTED),
("via_mqtt", BIT_VIA_MQTT),
("is_favorite", BIT_IS_FAVORITE),
("is_ignored", BIT_IS_IGNORED),
("has_user", BIT_HAS_USER),
("is_licensed", BIT_IS_LICENSED),
("is_unmessagable", BIT_IS_UNMESSAGABLE),
("has_is_unmessagable", BIT_HAS_IS_UNMESSAGABLE),
)
def _pack_bitfield(bf: dict[str, bool]) -> int:
out = 0
for key, shift in BITFIELD_LAYOUT:
if bf.get(key, False):
out |= (1 << shift)
return out
def _validate_node(node: dict[str, Any]) -> None:
"""Friendly errors so hand-editors get clear feedback."""
if "num" not in node or not isinstance(node["num"], str):
raise ValueError(f"node missing/invalid 'num' (must be hex string): {node!r}")
if "long_name" not in node:
raise ValueError(f"node {node['num']}: missing 'long_name'")
if len(node["long_name"]) > 24:
raise ValueError(
f"node {node['num']}: long_name {node['long_name']!r} is "
f"{len(node['long_name'])} chars; max 24 (nanopb max_size:25 minus NUL)"
)
if "short_name" in node:
# short_name max_size:5 (incl. NUL) → 4 bytes of content.
# Char count is irrelevant — emojis with variation selectors (e.g. ❄️ = 6 B)
# would slip past a `len(str) > 4` check. Always measure bytes.
b = node["short_name"].encode("utf-8")
if len(b) > 4:
raise ValueError(
f"node {node['num']}: short_name {node['short_name']!r} is "
f"{len(b)} bytes UTF-8; max 4 (nanopb max_size:5 minus NUL)"
)
pk = node.get("public_key_hex", "")
if pk and len(pk) != 64:
raise ValueError(
f"node {node['num']}: public_key_hex must be 64 hex chars or empty; "
f"got {len(pk)} chars"
)
if pk:
try:
bytes.fromhex(pk)
except ValueError as e:
raise ValueError(f"node {node['num']}: public_key_hex is not valid hex: {e}")
def _resolve_time(
node: dict[str, Any],
field_absolute: str,
field_offset: str,
now_epoch: int,
) -> int:
"""If `field_absolute` is set, use it; else compute `now_epoch - offset`."""
if field_absolute in node and node[field_absolute] is not None:
return int(node[field_absolute])
offset = node.get(field_offset, 0)
return max(0, int(now_epoch) - int(offset))
def _build_node_info_lite(node: dict[str, Any], now_epoch: int) -> NodeInfoLite:
_validate_node(node)
info = NodeInfoLite()
info.num = int(node["num"], 16) if isinstance(node["num"], str) else int(node["num"])
info.long_name = node.get("long_name", "")
info.short_name = node.get("short_name", "")
# Enum lookups will raise ValueError on unknown names — that's exactly what we want.
info.hw_model = HardwareModel.Value(node.get("hw_model", "UNSET"))
info.role = Config.DeviceConfig.Role.Value(node.get("role", "CLIENT"))
pk_hex = node.get("public_key_hex", "")
if pk_hex:
info.public_key = bytes.fromhex(pk_hex)
info.snr = float(node.get("snr", 0.0))
info.channel = int(node.get("channel", 0))
if "hops_away" in node:
# `optional uint32 hops_away = 9;` — in Python protobuf, assigning the
# field implicitly sets HasField("hops_away") to True. No has_hops_away
# setter exists (unlike the C++ nanopb-generated header).
info.hops_away = int(node["hops_away"])
info.next_hop = int(node.get("next_hop", 0))
info.last_heard = _resolve_time(node, "last_heard", "last_heard_offset_sec", now_epoch)
info.bitfield = _pack_bitfield(node.get("bitfield", {}))
return info
def _build_position_entry(num: int, pos: dict[str, Any], now_epoch: int) -> NodePositionEntry:
entry = NodePositionEntry()
entry.num = num
pl = PositionLite()
# Firmware stores lat/long as int32 in 1e-7 degrees.
pl.latitude_i = int(round(float(pos["latitude"]) * 1e7))
pl.longitude_i = int(round(float(pos["longitude"]) * 1e7))
pl.altitude = int(pos.get("altitude", 0))
pl.time = _resolve_time(pos, "time", "time_offset_sec", now_epoch)
pl.location_source = Position.LocSource.Value(pos.get("location_source", "LOC_UNSET"))
entry.position.CopyFrom(pl)
return entry
def _build_telemetry_entry(num: int, tel: dict[str, Any]) -> NodeTelemetryEntry:
entry = NodeTelemetryEntry()
entry.num = num
dm = DeviceMetrics()
if "battery_level" in tel:
dm.battery_level = int(tel["battery_level"])
if "voltage" in tel:
dm.voltage = float(tel["voltage"])
if "channel_utilization" in tel:
dm.channel_utilization = float(tel["channel_utilization"])
if "air_util_tx" in tel:
dm.air_util_tx = float(tel["air_util_tx"])
if "uptime_seconds" in tel:
dm.uptime_seconds = int(tel["uptime_seconds"])
entry.device_metrics.CopyFrom(dm)
return entry
def _build_environment_entry(num: int, env: dict[str, Any]) -> NodeEnvironmentEntry:
entry = NodeEnvironmentEntry()
entry.num = num
em = EnvironmentMetrics()
if "temperature" in env:
em.temperature = float(env["temperature"])
if "relative_humidity" in env:
em.relative_humidity = float(env["relative_humidity"])
if "barometric_pressure" in env:
em.barometric_pressure = float(env["barometric_pressure"])
if "iaq" in env:
em.iaq = int(env["iaq"])
entry.environment_metrics.CopyFrom(em)
return entry
def _build_status_entry(num: int, status: dict[str, Any]) -> NodeStatusEntry:
# `StatusMessage` (mesh.proto:1445) has a single `string status` field.
entry = NodeStatusEntry()
entry.num = num
sm = StatusMessage()
if "status" in status:
sm.status = str(status["status"])
entry.status.CopyFrom(sm)
return entry
def compile_jsonl_to_proto(jsonl_path: pathlib.Path, now_epoch: int) -> bytes:
"""Read a seed JSONL and return the encoded NodeDatabase bytes."""
lines = jsonl_path.read_text(encoding="utf-8").splitlines()
if not lines:
raise ValueError(f"{jsonl_path} is empty")
meta_line = lines[0]
meta_obj = json.loads(meta_line)
meta = meta_obj.get("_meta", {})
version = meta.get("version")
if version != 25:
raise ValueError(
f"{jsonl_path}: meta version is {version!r}; this compiler "
f"requires version=25. Regenerate the seed with the matching tooling."
)
db = NodeDatabase()
db.version = 25
for ln, raw in enumerate(lines[1:], start=2):
raw = raw.strip()
if not raw:
continue
try:
node = json.loads(raw)
except json.JSONDecodeError as e:
raise ValueError(f"{jsonl_path}:{ln} JSON parse error: {e}")
num = int(node["num"], 16) if isinstance(node["num"], str) else int(node["num"])
# Header
info = _build_node_info_lite(node, now_epoch)
db.nodes.append(info)
# Satellites (nullable)
if node.get("position"):
db.positions.append(_build_position_entry(num, node["position"], now_epoch))
if node.get("telemetry"):
db.telemetry.append(_build_telemetry_entry(num, node["telemetry"]))
if node.get("environment"):
db.environment.append(_build_environment_entry(num, node["environment"]))
if node.get("status"):
db.status.append(_build_status_entry(num, node["status"]))
return db.SerializeToString()
def main(argv: list[str]) -> int:
p = argparse.ArgumentParser(
description="Compile a seed JSONL into a binary v25 NodeDatabase proto.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
p.add_argument("--in", dest="in_path", required=True, help="Input seed JSONL.")
p.add_argument("--out", required=True, help="Output binary .proto path.")
p.add_argument(
"--now-epoch",
type=int,
default=None,
help="Pin 'now' to this Unix epoch (for byte-identical CI). Default: time.time().",
)
args = p.parse_args(argv)
in_path = pathlib.Path(args.in_path)
if not in_path.is_file():
print(f"input not found: {in_path}", file=sys.stderr)
return 2
now_epoch = args.now_epoch if args.now_epoch is not None else int(time.time())
try:
encoded = compile_jsonl_to_proto(in_path, now_epoch)
except ValueError as e:
print(f"ERROR: {e}", file=sys.stderr)
return 3
out_path = pathlib.Path(args.out)
out_path.parent.mkdir(parents=True, exist_ok=True)
out_path.write_bytes(encoded)
print(
f"compiled {in_path} -> {out_path} ({len(encoded)} bytes, now_epoch={now_epoch})",
file=sys.stderr,
)
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
+95
View File
@@ -0,0 +1,95 @@
import sys
import os
import json
from github import Github
def parseFile(path):
with open(path, "r") as f:
data = json.loads(f)
for file in data["files"]:
if file["name"].endswith(".bin"):
return file["name"], file["bytes"]
if len(sys.argv) != 4:
print(f"expected usage: {sys.argv[0]} <PR number> <path to old-manifests> <path to new-manifests>")
sys.exit(1)
pr_number = int(sys.argv[1])
token = os.getenv("GITHUB_TOKEN")
if not token:
raise EnvironmentError("GITHUB_TOKEN not found in environment.")
repo_name = os.getenv("GITHUB_REPOSITORY") # "owner/repo"
if not repo_name:
raise EnvironmentError("GITHUB_REPOSITORY not found in environment.")
oldFiles = sys.argv[2]
old = set(os.path.join(oldFiles, f) for f in os.listdir(oldFiles) if os.path.isfile(f))
newFiles = sys.argv[3]
new = set(os.path.join(newFiles, f) for f in os.listdir(newFiles) if os.path.isfile(f))
startMarkdown = "# Target Size Changes\n\n"
markdown = ""
newlyIntroduced = new - old
if len(newlyIntroduced) > 0:
markdown += "## Newly Introduced Targets\n\n"
# create a table
markdown += "| File | Size |\n"
markdown += "| ---- | ---- |\n"
for f in newlyIntroduced:
name, size = parseFile(f)
markdown += f"| `{name}` | {size}b |\n"
# do not log removed targets
# PRs only run a small subset of builds, so removed targets are not meaningful
# since they are very likely to just be not ran in PR CI
both = old & new
degradations = []
improvements = []
for f in both:
oldName, oldSize = parseFile(f)
_, newSize = parseFile(f)
if oldSize != newSize:
if newSize < oldSize:
improvements.append((oldName, oldSize, newSize))
else:
degradations.append((oldName, oldSize, newSize))
if len(degradations) > 0:
markdown += "\n## Degradation\n\n"
# create a table
markdown += "| File | Difference | Old Size | New Size |\n"
markdown += "| ---- | ---------- | -------- | -------- |\n"
for oldName, oldSize, newSize in degradations:
markdown += f"| `{oldName}` | **{oldSize - newSize}b** | {oldSize}b | {newSize}b |\n"
if len(improvements) > 0:
markdown += "\n## Improvement\n\n"
# create a table
markdown += "| File | Difference | Old Size | New Size |\n"
markdown += "| ---- | ---------- | -------- | -------- |\n"
for oldName, oldSize, newSize in improvements:
markdown += f"| `{oldName}` | **{oldSize - newSize}b** | {oldSize}b | {newSize}b |\n"
if len(markdown) == 0:
markdown = "No changes in target sizes detected."
g = Github(token)
repo = g.get_repo(repo_name)
pr = repo.get_pull(pr_number)
existing_comment = None
for comment in pr.get_issue_comments():
if comment.body.startswith(startMarkdown):
existing_comment = comment
break
final_markdown = startMarkdown + markdown
if existing_comment:
existing_comment.edit(body=final_markdown)
else:
pr.create_issue_comment(body=final_markdown)
-171
View File
@@ -1,171 +0,0 @@
#!/usr/bin/env python3
"""Compare firmware size reports and generate a markdown summary.
Usage:
size_report.py <new_sizes.json> [--baseline <label>:<old_sizes.json>]...
Examples:
# Compare PR against develop and master baselines
size_report.py pr.json --baseline develop:develop.json --baseline master:master.json
# Single baseline comparison
size_report.py pr.json --baseline develop:develop.json
# No baselines — shows sizes with blank delta columns
size_report.py pr.json
"""
import argparse
import json
import sys
def load_sizes(path):
with open(path) as f:
return json.load(f)
def format_delta(n):
"""Format byte delta with sign and human-friendly suffix."""
sign = "+" if n > 0 else ""
if abs(n) >= 1024:
return f"{sign}{n:,} ({sign}{n / 1024:.1f} KB)"
return f"{sign}{n:,}"
def generate_markdown(new_sizes, baselines, top_n=5):
"""Generate a single table with current size and delta columns per baseline.
baselines: list of (label, old_sizes_dict), may be empty
"""
labels = [label for label, _ in baselines]
# Build rows: (board, current_size, [(delta, abs_delta) per baseline])
rows = []
for board in sorted(new_sizes):
current = new_sizes[board]
deltas = []
for _, old_sizes in baselines:
old = old_sizes.get(board)
if old is not None:
d = current - old
deltas.append((d, abs(d)))
else:
deltas.append((None, 0))
# Sort key: max abs delta across baselines (biggest changes first)
max_abs = max((ad for _, ad in deltas), default=0)
rows.append((board, current, deltas, max_abs))
rows.sort(key=lambda r: r[3], reverse=True)
# Summary line
sections = []
summary_parts = [f"{len(new_sizes)} targets"]
for i, (label, old_sizes) in enumerate(baselines):
increases = sum(
1 for _, _, deltas, _ in rows if deltas[i][0] is not None and deltas[i][0] > 0
)
decreases = sum(
1 for _, _, deltas, _ in rows if deltas[i][0] is not None and deltas[i][0] < 0
)
net = sum(
deltas[i][0] for _, _, deltas, _ in rows if deltas[i][0] is not None
)
parts = []
if increases:
parts.append(f"{increases} increased")
if decreases:
parts.append(f"{decreases} decreased")
if parts:
parts.append(f"net {format_delta(net)}")
summary_parts.append(f"vs `{label}`: {', '.join(parts)}")
else:
summary_parts.append(f"vs `{label}`: no changes")
if not baselines:
summary_parts.append("no baseline available yet")
sections.append(f"**{' | '.join(summary_parts)}**\n")
# Table header
header = "| Target | Size |"
separator = "|--------|-----:|"
for label in labels:
header += f" vs `{label}` |"
separator += "----------:|"
sections.append(header)
sections.append(separator)
def format_row(board, current, deltas):
row = f"| `{board}` | {current:,} |"
for d, _ in deltas:
if d is None:
row += " |"
elif d == 0:
row += " 0 |"
else:
icon = "📈" if d > 0 else "📉"
row += f" {icon} {format_delta(d)} |"
return row
# Top N rows always visible
top = rows[:top_n]
for board, current, deltas, _ in top:
sections.append(format_row(board, current, deltas))
# Remaining rows in expandable section
rest = rows[top_n:]
if rest:
sections.append("")
sections.append(
f"<details><summary>Show {len(rest)} more target(s)</summary>\n"
)
sections.append(header)
sections.append(separator)
for board, current, deltas, _ in rest:
sections.append(format_row(board, current, deltas))
sections.append("\n</details>")
sections.append("")
return "\n".join(sections)
def main():
parser = argparse.ArgumentParser(description="Compare firmware size reports")
parser.add_argument("new_sizes", help="Path to new sizes JSON")
parser.add_argument(
"--baseline",
action="append",
default=[],
metavar="LABEL:PATH",
help="Baseline to compare against (e.g. develop:develop.json)",
)
parser.add_argument(
"--top",
type=int,
default=5,
help="Number of top changes to show before collapsing (default: 5)",
)
args = parser.parse_args()
new_sizes = load_sizes(args.new_sizes)
# Silence output when no targets were built — repo maintainer choice
if not new_sizes:
return
baselines = []
for b in args.baseline:
if ":" not in b:
print(f"Error: baseline must be LABEL:PATH, got '{b}'", file=sys.stderr)
sys.exit(1)
label, path = b.split(":", 1)
baselines.append((label, load_sizes(path)))
md = generate_markdown(new_sizes, baselines, top_n=args.top)
print(md)
if __name__ == "__main__":
main()
-262
View File
@@ -1,262 +0,0 @@
#!/usr/bin/env python3
"""Tests for bin/collect_sizes.py and bin/size_report.py."""
import json
import os
import subprocess
import sys
import tempfile
SCRIPTS_DIR = os.path.join(os.path.dirname(__file__), "..", "bin")
def make_manifest(target, firmware_bytes, extra_files=None):
"""Create a minimal .mt.json manifest dict."""
files = [{"name": f"firmware-{target}-2.6.0.bin", "bytes": firmware_bytes}]
if extra_files:
files.extend(extra_files)
return {
"platformioTarget": target,
"version": "2.6.0.test",
"files": files,
}
def write_manifests(tmpdir, manifests):
"""Write manifest dicts as .mt.json files into tmpdir."""
for target, data in manifests.items():
path = os.path.join(tmpdir, f"firmware-{target}.mt.json")
with open(path, "w") as f:
json.dump(data, f)
def run_script(script, args):
"""Run a Python script and return (returncode, stdout, stderr)."""
result = subprocess.run(
[sys.executable, os.path.join(SCRIPTS_DIR, script)] + args,
capture_output=True,
text=True,
)
return result.returncode, result.stdout, result.stderr
def test_collect_sizes_basic():
"""collect_sizes picks up firmware-*.bin entries from manifests."""
with tempfile.TemporaryDirectory() as tmpdir:
outfile = os.path.join(tmpdir, "sizes.json")
manifests = {
"heltec-v3": make_manifest("heltec-v3", 1048576),
"rak4631": make_manifest("rak4631", 524288),
"tbeam": make_manifest("tbeam", 786432),
}
write_manifests(tmpdir, manifests)
rc, stdout, stderr = run_script("collect_sizes.py", [tmpdir, outfile])
assert rc == 0, f"collect_sizes failed: {stderr}"
assert "3 targets" in stdout
with open(outfile) as f:
sizes = json.load(f)
assert sizes == {"heltec-v3": 1048576, "rak4631": 524288, "tbeam": 786432}
def test_collect_sizes_fallback_bin():
"""collect_sizes falls back to non-firmware-prefixed .bin if no firmware-*.bin."""
with tempfile.TemporaryDirectory() as tmpdir:
outfile = os.path.join(tmpdir, "sizes.json")
# Manifest with only a generic .bin (no firmware- prefix)
data = {
"platformioTarget": "custom-board",
"files": [
{"name": "littlefs-custom-board.bin", "bytes": 100000},
{"name": "custom-board.bin", "bytes": 500000},
],
}
path = os.path.join(tmpdir, "firmware-custom-board.mt.json")
with open(path, "w") as f:
json.dump(data, f)
rc, stdout, stderr = run_script("collect_sizes.py", [tmpdir, outfile])
assert rc == 0, f"collect_sizes failed: {stderr}"
with open(outfile) as f:
sizes = json.load(f)
assert sizes == {"custom-board": 500000}
def test_collect_sizes_skips_ota_littlefs():
"""collect_sizes ignores ota/littlefs/bleota .bin files in fallback."""
with tempfile.TemporaryDirectory() as tmpdir:
outfile = os.path.join(tmpdir, "sizes.json")
data = {
"platformioTarget": "board-x",
"files": [
{"name": "littlefs-board-x.bin", "bytes": 100000},
{"name": "bleota-board-x.bin", "bytes": 50000},
{"name": "mt-board-x-ota.bin", "bytes": 60000},
],
}
path = os.path.join(tmpdir, "firmware-board-x.mt.json")
with open(path, "w") as f:
json.dump(data, f)
rc, stdout, stderr = run_script("collect_sizes.py", [tmpdir, outfile])
assert rc == 0
with open(outfile) as f:
sizes = json.load(f)
# No valid firmware .bin found, board should be absent
assert sizes == {}
def test_collect_sizes_ignores_non_mt_json():
"""collect_sizes skips non .mt.json files."""
with tempfile.TemporaryDirectory() as tmpdir:
outfile = os.path.join(tmpdir, "sizes.json")
# Write a valid manifest
manifests = {"rak4631": make_manifest("rak4631", 500000)}
write_manifests(tmpdir, manifests)
# Write a decoy file
with open(os.path.join(tmpdir, "readme.txt"), "w") as f:
f.write("not a manifest")
rc, stdout, stderr = run_script("collect_sizes.py", [tmpdir, outfile])
assert rc == 0
with open(outfile) as f:
sizes = json.load(f)
assert list(sizes.keys()) == ["rak4631"]
def test_size_report_no_baseline():
"""size_report with no baselines shows sizes only."""
with tempfile.TemporaryDirectory() as tmpdir:
sizes_file = os.path.join(tmpdir, "new.json")
with open(sizes_file, "w") as f:
json.dump({"heltec-v3": 1000000, "rak4631": 500000}, f)
rc, stdout, stderr = run_script("size_report.py", [sizes_file])
assert rc == 0, f"size_report failed: {stderr}"
assert "2 targets" in stdout
assert "no baseline available yet" in stdout
assert "`heltec-v3`" in stdout
assert "`rak4631`" in stdout
def test_size_report_with_baseline():
"""size_report shows deltas against a baseline."""
with tempfile.TemporaryDirectory() as tmpdir:
new_file = os.path.join(tmpdir, "new.json")
old_file = os.path.join(tmpdir, "old.json")
with open(new_file, "w") as f:
json.dump({"heltec-v3": 1050000, "rak4631": 500000, "tbeam": 800000}, f)
with open(old_file, "w") as f:
json.dump({"heltec-v3": 1000000, "rak4631": 500000, "tbeam": 810000}, f)
rc, stdout, stderr = run_script(
"size_report.py", [new_file, "--baseline", f"develop:{old_file}"]
)
assert rc == 0, f"size_report failed: {stderr}"
assert "3 targets" in stdout
assert "1 increased" in stdout
assert "1 decreased" in stdout
# heltec-v3 grew by 50000
assert "📈" in stdout
# tbeam shrank by 10000
assert "📉" in stdout
# rak4631 unchanged
assert "vs `develop`" in stdout
def test_size_report_multiple_baselines():
"""size_report handles multiple baselines."""
with tempfile.TemporaryDirectory() as tmpdir:
new_file = os.path.join(tmpdir, "new.json")
dev_file = os.path.join(tmpdir, "develop.json")
master_file = os.path.join(tmpdir, "master.json")
with open(new_file, "w") as f:
json.dump({"board-a": 100000}, f)
with open(dev_file, "w") as f:
json.dump({"board-a": 95000}, f)
with open(master_file, "w") as f:
json.dump({"board-a": 90000}, f)
rc, stdout, stderr = run_script(
"size_report.py",
[new_file, "--baseline", f"develop:{dev_file}", "--baseline", f"master:{master_file}"],
)
assert rc == 0, f"size_report failed: {stderr}"
assert "vs `develop`" in stdout
assert "vs `master`" in stdout
def test_size_report_new_target_no_baseline_entry():
"""size_report handles targets not present in baseline (new boards)."""
with tempfile.TemporaryDirectory() as tmpdir:
new_file = os.path.join(tmpdir, "new.json")
old_file = os.path.join(tmpdir, "old.json")
with open(new_file, "w") as f:
json.dump({"new-board": 300000, "existing": 500000}, f)
with open(old_file, "w") as f:
json.dump({"existing": 500000}, f)
rc, stdout, stderr = run_script(
"size_report.py", [new_file, "--baseline", f"develop:{old_file}"]
)
assert rc == 0, f"size_report failed: {stderr}"
assert "`new-board`" in stdout
assert "no changes" in stdout # only existing is compared, delta=0
def test_size_report_all_unchanged():
"""size_report shows 'no changes' when all sizes match."""
with tempfile.TemporaryDirectory() as tmpdir:
sizes_file = os.path.join(tmpdir, "sizes.json")
with open(sizes_file, "w") as f:
json.dump({"board-a": 100000, "board-b": 200000}, f)
rc, stdout, stderr = run_script(
"size_report.py", [sizes_file, "--baseline", f"develop:{sizes_file}"]
)
assert rc == 0, f"size_report failed: {stderr}"
assert "no changes" in stdout
def test_collect_sizes_bad_args():
"""collect_sizes exits with error on wrong arg count."""
rc, stdout, stderr = run_script("collect_sizes.py", [])
assert rc == 1
assert "Usage" in stderr
def test_size_report_bad_baseline_format():
"""size_report exits with error on malformed --baseline."""
with tempfile.TemporaryDirectory() as tmpdir:
sizes_file = os.path.join(tmpdir, "sizes.json")
with open(sizes_file, "w") as f:
json.dump({"x": 1}, f)
rc, stdout, stderr = run_script(
"size_report.py", [sizes_file, "--baseline", "no-colon-here"]
)
assert rc == 1
assert "LABEL:PATH" in stderr
if __name__ == "__main__":
tests = [v for k, v in globals().items() if k.startswith("test_")]
passed = 0
failed = 0
for test in tests:
try:
test()
print(f" PASS: {test.__name__}")
passed += 1
except AssertionError as e:
print(f" FAIL: {test.__name__}: {e}")
failed += 1
except Exception as e:
print(f" ERROR: {test.__name__}: {type(e).__name__}: {e}")
failed += 1
print(f"\n{passed} passed, {failed} failed out of {passed + failed}")
sys.exit(1 if failed else 0)
+1 -1
View File
@@ -7,7 +7,7 @@
"extra_flags": [
"-D CDEBYTE_EORA_S3",
"-D ARDUINO_USB_CDC_ON_BOOT=1",
"-D ARDUINO_USB_MODE=1",
"-D ARDUINO_USB_MODE=0",
"-D ARDUINO_RUNNING_CORE=1",
"-D ARDUINO_EVENT_RUNNING_CORE=1",
"-D BOARD_HAS_PSRAM"
+1 -1
View File
@@ -6,7 +6,7 @@
"core": "esp32",
"extra_flags": [
"-DARDUINO_USB_CDC_ON_BOOT=1",
"-DARDUINO_USB_MODE=1",
"-DARDUINO_USB_MODE=0",
"-DARDUINO_RUNNING_CORE=1",
"-DARDUINO_EVENT_RUNNING_CORE=1",
"-DBOARD_HAS_PSRAM"
+1 -1
View File
@@ -8,7 +8,7 @@
"extra_flags": [
"-DBOARD_HAS_PSRAM",
"-DARDUINO_USB_CDC_ON_BOOT=1",
"-DARDUINO_USB_MODE=1",
"-DARDUINO_USB_MODE=0",
"-DARDUINO_RUNNING_CORE=1",
"-DARDUINO_EVENT_RUNNING_CORE=1"
],
-54
View File
@@ -1,54 +0,0 @@
{
"build": {
"arduino": {
"ldscript": "nrf52840_s140_v6.ld"
},
"core": "nRF5",
"cpu": "cortex-m4",
"extra_flags": "-DNRF52840_XXAA",
"f_cpu": "64000000L",
"hwids": [
["0x239A", "0x4405"],
["0x239A", "0x0029"],
["0x239A", "0x002A"],
["0x2886", "0x1667"]
],
"usb_product": "HT-n5262",
"mcu": "nrf52840",
"variant": "heltec_mesh_node_t1",
"variants_dir": "variants",
"bsp": {
"name": "adafruit"
},
"softdevice": {
"sd_flags": "-DS140",
"sd_name": "s140",
"sd_version": "6.1.1",
"sd_fwid": "0x00B6"
},
"bootloader": {
"settings_addr": "0xFF000"
}
},
"connectivity": ["bluetooth"],
"debug": {
"jlink_device": "nRF52840_xxAA",
"onboard_tools": ["jlink"],
"svd_path": "nrf52840.svd",
"openocd_target": "nrf52840-mdk-rs"
},
"frameworks": ["arduino"],
"name": "Heltec Mesh Node T1",
"upload": {
"maximum_ram_size": 248832,
"maximum_size": 815104,
"speed": 115200,
"protocol": "nrfutil",
"protocols": ["jlink", "nrfjprog", "nrfutil", "stlink"],
"use_1200bps_touch": true,
"require_upload_port": true,
"wait_for_upload_port": true
},
"url": "https://heltec.org",
"vendor": "Heltec"
}
+1 -1
View File
@@ -9,7 +9,7 @@
"extra_flags": [
"-DBOARD_HAS_PSRAM",
"-DARDUINO_USB_CDC_ON_BOOT=1",
"-DARDUINO_USB_MODE=1",
"-DARDUINO_USB_MODE=0",
"-DARDUINO_RUNNING_CORE=1",
"-DARDUINO_EVENT_RUNNING_CORE=1"
],
+1 -1
View File
@@ -9,7 +9,7 @@
"extra_flags": [
"-DBOARD_HAS_PSRAM",
"-DARDUINO_USB_CDC_ON_BOOT=1",
"-DARDUINO_USB_MODE=1",
"-DARDUINO_USB_MODE=0",
"-DARDUINO_RUNNING_CORE=1",
"-DARDUINO_EVENT_RUNNING_CORE=1"
],
+1 -1
View File
@@ -9,7 +9,7 @@
"extra_flags": [
"-DBOARD_HAS_PSRAM",
"-DARDUINO_USB_CDC_ON_BOOT=1",
"-DARDUINO_USB_MODE=1",
"-DARDUINO_USB_MODE=0",
"-DARDUINO_RUNNING_CORE=1",
"-DARDUINO_EVENT_RUNNING_CORE=1"
],
+1 -1
View File
@@ -8,7 +8,7 @@
"extra_flags": [
"-DHELTEC_WIRELESS_TRACKER",
"-DARDUINO_USB_CDC_ON_BOOT=1",
"-DARDUINO_USB_MODE=1",
"-DARDUINO_USB_MODE=0",
"-DARDUINO_RUNNING_CORE=1",
"-DARDUINO_EVENT_RUNNING_CORE=1"
],
+1 -1
View File
@@ -7,7 +7,7 @@
"core": "esp32",
"extra_flags": [
"-DARDUINO_USB_CDC_ON_BOOT=1",
"-DARDUINO_USB_MODE=1",
"-DARDUINO_USB_MODE=0",
"-DARDUINO_RUNNING_CORE=1",
"-DARDUINO_EVENT_RUNNING_CORE=1"
],
-26
View File
@@ -1,26 +0,0 @@
{
"build": {
"cpu": "cortex-m33",
"f_cpu": "128000000L",
"mcu": "nrf54l15",
"zephyr": {
"variant": "nrf54l15dk/nrf54l15/cpuapp"
}
},
"connectivity": ["bluetooth"],
"debug": {
"default_tools": ["jlink"],
"jlink_device": "nRF54L15_M33",
"svd_path": "nrf54l15.svd"
},
"frameworks": ["zephyr"],
"name": "Nordic nRF54L15-DK (PCA10156)",
"upload": {
"maximum_ram_size": 262144,
"maximum_size": 1572864,
"protocol": "jlink",
"protocols": ["jlink"]
},
"url": "https://www.nordicsemi.com/Products/nRF54L15",
"vendor": "Nordic Semiconductor"
}
+1 -1
View File
@@ -8,7 +8,7 @@
"extra_flags": [
"-DBOARD_HAS_PSRAM",
"-DARDUINO_USB_CDC_ON_BOOT=1",
"-DARDUINO_USB_MODE=1",
"-DARDUINO_USB_MODE=0",
"-DARDUINO_RUNNING_CORE=1",
"-DARDUINO_EVENT_RUNNING_CORE=0"
],
-41
View File
@@ -1,41 +0,0 @@
{
"build": {
"arduino": {
"ldscript": "esp32s3_out.ld",
"memory_type": "qio_opi"
},
"core": "esp32",
"extra_flags": [
"-DBOARD_HAS_PSRAM",
"-DARDUINO_USB_CDC_ON_BOOT=1",
"-DARDUINO_USB_MODE=1",
"-DARDUINO_RUNNING_CORE=1",
"-DARDUINO_EVENT_RUNNING_CORE=0"
],
"f_cpu": "240000000L",
"f_flash": "80000000L",
"flash_mode": "qio",
"hwids": [["0x303A", "0x1001"]],
"mcu": "esp32s3",
"variant": "station-g3"
},
"connectivity": ["wifi", "bluetooth", "lora"],
"debug": {
"default_tool": "esp-builtin",
"onboard_tools": ["esp-builtin"],
"openocd_target": "esp32s3.cfg"
},
"frameworks": ["arduino", "espidf"],
"name": "BQ Station G3",
"upload": {
"flash_size": "16MB",
"maximum_ram_size": 327680,
"maximum_size": 16777216,
"use_1200bps_touch": true,
"wait_for_upload_port": true,
"require_upload_port": true,
"speed": 921600
},
"url": "",
"vendor": "BQ Consulting"
}
+1 -1
View File
@@ -9,7 +9,7 @@
"-DBOARD_HAS_PSRAM",
"-DLILYGO_TBEAM_1W",
"-DARDUINO_USB_CDC_ON_BOOT=1",
"-DARDUINO_USB_MODE=1",
"-DARDUINO_USB_MODE=0",
"-DARDUINO_RUNNING_CORE=1",
"-DARDUINO_EVENT_RUNNING_CORE=1"
],
+1 -1
View File
@@ -8,7 +8,7 @@
"extra_flags": [
"-DBOARD_HAS_PSRAM",
"-DARDUINO_USB_CDC_ON_BOOT=1",
"-DARDUINO_USB_MODE=1",
"-DARDUINO_USB_MODE=0",
"-DARDUINO_RUNNING_CORE=1",
"-DARDUINO_EVENT_RUNNING_CORE=1"
],
+1 -1
View File
@@ -8,7 +8,7 @@
"-DBOARD_HAS_PSRAM",
"-DLILYGO_TBEAM_S3_CORE",
"-DARDUINO_USB_CDC_ON_BOOT=1",
"-DARDUINO_USB_MODE=1",
"-DARDUINO_USB_MODE=0",
"-DARDUINO_RUNNING_CORE=1",
"-DARDUINO_EVENT_RUNNING_CORE=1"
],
+1 -1
View File
@@ -7,7 +7,7 @@
"extra_flags": [
"-DLILYGO_T3S3_V1",
"-DARDUINO_USB_CDC_ON_BOOT=1",
"-DARDUINO_USB_MODE=1",
"-DARDUINO_USB_MODE=0",
"-DARDUINO_RUNNING_CORE=1",
"-DARDUINO_EVENT_RUNNING_CORE=1",
"-DBOARD_HAS_PSRAM"
+1 -1
View File
@@ -10,7 +10,7 @@
"-DBOARD_HAS_PSRAM",
"-DUNPHONE_SPIN=9",
"-DARDUINO_USB_CDC_ON_BOOT=1",
"-DARDUINO_USB_MODE=1",
"-DARDUINO_USB_MODE=0",
"-DARDUINO_RUNNING_CORE=1",
"-DARDUINO_EVENT_RUNNING_CORE=1"
],
+50
View File
@@ -0,0 +1,50 @@
{
"build": {
"arduino": {
"ldscript": "nrf52832_s132_v6.ld"
},
"core": "nRF5",
"cpu": "cortex-m4",
"extra_flags": "-DNRF52832_XXAA -DNRF52",
"f_cpu": "64000000L",
"hwids": [
["0x239A", "0x8029"],
["0x239A", "0x0029"],
["0x239A", "0x002A"],
["0x239A", "0x802A"]
],
"usb_product": "Feather nRF52832 Express",
"mcu": "nrf52832",
"variant": "WisCore_RAK4600_Board",
"bsp": {
"name": "adafruit"
},
"softdevice": {
"sd_flags": "-DS132",
"sd_name": "s132",
"sd_version": "6.1.1",
"sd_fwid": "0x00B7"
},
"zephyr": {
"variant": "nrf52_adafruit_feather"
}
},
"connectivity": ["bluetooth"],
"debug": {
"jlink_device": "nRF52832_xxAA",
"svd_path": "nrf52.svd",
"openocd_target": "nrf52840-mdk-rs"
},
"frameworks": ["arduino", "zephyr"],
"name": "Adafruit Bluefruit nRF52832 Feather",
"upload": {
"maximum_ram_size": 65536,
"maximum_size": 524288,
"require_upload_port": true,
"speed": 115200,
"protocol": "nrfutil",
"protocols": ["jlink", "nrfjprog", "nrfutil", "stlink"]
},
"url": "https://www.adafruit.com/product/3406",
"vendor": "Adafruit"
}
-1
View File
@@ -14,7 +14,6 @@ Build-Depends: debhelper-compat (= 13),
g++,
pkg-config,
libyaml-cpp-dev,
libjsoncpp-dev,
libgpiod-dev,
libbluetooth-dev,
libusb-1.0-0-dev,
-7
View File
@@ -1,7 +0,0 @@
# Name, Type, SubType, Offset, Size, Flags
nvs, data, nvs, 0x9000, 0x5000,
otadata, data, ota, 0xe000, 0x2000,
app0, app, ota_0, 0x10000, 0x640000,
app1, app, ota_1, 0x650000,0x640000,
spiffs, data, spiffs, 0xc90000,0x360000,
coredump, data, coredump,0xFF0000,0x10000,
1 # Name Type SubType Offset Size Flags
2 nvs data nvs 0x9000 0x5000
3 otadata data ota 0xe000 0x2000
4 app0 app ota_0 0x10000 0x640000
5 app1 app ota_1 0x650000 0x640000
6 spiffs data spiffs 0xc90000 0x360000
7 coredump data coredump 0xFF0000 0x10000
-7
View File
@@ -1,7 +0,0 @@
# Name, Type, SubType, Offset, Size, Flags
nvs, data, nvs, 0x9000, 0x5000,
otadata, data, ota, 0xe000, 0x2000,
app0, app, ota_0, 0x10000, 0x330000,
app1, app, ota_1, 0x340000,0x330000,
spiffs, data, spiffs, 0x670000,0x180000,
coredump, data, coredump,0x7F0000,0x10000,
1 # Name Type SubType Offset Size Flags
2 nvs data nvs 0x9000 0x5000
3 otadata data ota 0xe000 0x2000
4 app0 app ota_0 0x10000 0x330000
5 app1 app ota_1 0x340000 0x330000
6 spiffs data spiffs 0x670000 0x180000
7 coredump data coredump 0x7F0000 0x10000
+14 -3
View File
@@ -70,6 +70,17 @@ def esp32_create_combined_bin(source, target, env):
env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", esp32_create_combined_bin)
# Enable Newlib Nano formatting to save space
# ...but allow printf float support (compromise)
env.Append(LINKFLAGS=["--specs=nano.specs", "-u", "_printf_float"])
esp32_kind = env.GetProjectOption("custom_esp32_kind")
if esp32_kind == "esp32":
# Free up some IRAM by removing auxiliary SPI flash chip drivers.
# Wrapped stub symbols are defined in src/platform/esp32/iram-quirk.c.
env.Append(
LINKFLAGS=[
"-Wl,--wrap=esp_flash_chip_gd",
"-Wl,--wrap=esp_flash_chip_issi",
"-Wl,--wrap=esp_flash_chip_winbond",
]
)
else:
# For newer ESP32 targets, using newlib nano works better.
env.Append(LINKFLAGS=["--specs=nano.specs", "-u", "_printf_float"])
-23
View File
@@ -1,23 +0,0 @@
#!/usr/bin/env python3
# trunk-ignore-all(ruff/F821)
# trunk-ignore-all(flake8/F821): For SConstruct imports
# force linker response file instead of command line arguments
Import("env")
def wrap_with_tempfile(command_key):
command = env.get(command_key)
if not command or not isinstance(command, str):
return
if "TEMPFILE(" in command:
return
env.Replace(**{command_key: "${TEMPFILE('%s')}" % command})
# Force SCons to spill long commands into response files on this target.
env.Replace(MAXLINELENGTH=8192)
for key in ("LINKCOM", "CXXLINKCOM", "SHLINKCOM", "SHCXXLINKCOM"):
wrap_with_tempfile(key)
-148
View File
@@ -1,148 +0,0 @@
#!/usr/bin/env python3
# trunk-ignore-all(ruff/F821)
# trunk-ignore-all(flake8/F821)
#
# Whole-image LTO for nrf52840 (~-60KB; ~-23KB beyond src-only LTO), EXCEPT the objects
# that own interrupt/exception handlers.
#
# Every ISR is referenced only from the assembly vector table (gcc_startup_nrf52840.S),
# which LTO cannot see -> whole-program LTO judges the handlers dead, removes them, and
# the weak `b .` Default_Handler stubs prevail -> the IRQ lands in an infinite loop and the
# chip hangs (or the peripheral silently stalls). Compiling the handler-bearing objects
# WITHOUT LTO lets ordinary linking keep the strong handlers; everything else stays LTO'd:
# - framework core (/FrameworkArduino/, /cores/nRF5/): every nrfx ISR + the FreeRTOS
# SVC/PendSV port.
# - TinyUSB nrf port (Adafruit_TinyUSB_nrf.cpp): USBD_IRQHandler (USB data path).
# - library .cpp files that own a vector ISR (would otherwise be silently dropped):
# bluefruit.cpp -> SD_EVT/SWI2_EGU2 (SoftDevice BLE-event delivery -- advertising
# hangs without it)
# Wire_nRF52.cpp -> SPIM0/TWIM0 + SPIM1/TWIM1 (interrupt-driven I2C/SPI)
# PDM.cpp -> PDM_IRQHandler (PDM microphone)
# RotaryEncoder.cpp -> QDEC_IRQHandler (hardware quadrature/rotary encoder)
#
# A post-link guard (bottom of this file) fails the build if a critical handler was dropped
# anyway -- so a future deps bump or a new ISR-owning library becomes a red build, not a field
# hang. To hunt a dropped ISR by hand: nm the .elf for `_IRQHandler$` symbols marked `W`, then
# grep the libs/framework for who defines them.
#
# HW-validated: RAK4631 (SX1262) + muzi-base (LR1121).
import glob
import os
Import("env")
env.Append(LINKFLAGS=["-flto", "-flto-partition=1to1"])
# The -fno-lto re-compiles below run with the global env, which lacks the framework's
# bundled-library include dirs -- and those libs cross-include each other (Wire pulls in
# Adafruit_TinyUSB.h, which pulls in SPI.h, ...). Add every bundled-lib dir (+ its src/) so
# the re-compiles resolve without chasing headers one at a time.
_fw = env.PioPlatform().get_package_dir("framework-arduinoadafruitnrf52") or ""
_extra_inc = []
for _d in sorted(glob.glob(os.path.join(_fw, "libraries", "*"))):
if os.path.isdir(_d):
_extra_inc.append(_d)
if os.path.isdir(os.path.join(_d, "src")):
_extra_inc.append(os.path.join(_d, "src"))
FRAMEWORK = ("/FrameworkArduino/", "/cores/nRF5/")
USB_ISR = "Adafruit_TinyUSB_nrf" # USBD_IRQHandler
# Library .cpp files that define vector-table ISRs (the rest of their lib stays LTO'd):
LIB_ISR = ("/bluefruit.cpp", "/Wire_nRF52.cpp", "/PDM.cpp", "/RotaryEncoder.cpp")
def _no_lto(node):
try:
path = node.get_abspath()
except Exception:
path = str(node)
path = path.replace(
"\\", "/"
) # normalize Windows backslashes so matches work cross-platform
if (
USB_ISR in path
or any(s in path for s in FRAMEWORK)
or any(s in path for s in LIB_ISR)
):
return env.Object(
node,
CCFLAGS=env["CCFLAGS"] + ["-fno-lto"],
CPPPATH=env["CPPPATH"] + _extra_inc,
)
return node
env.AddBuildMiddleware(_no_lto)
# --- post-link guard: catch a dropped ISR handler at build time (CI footgun protection) ----
# After every link, fail the build if one of these critical vector-table handlers resolved to
# the weak `b .` Default_Handler stub -- i.e. LTO (or a deps bump, or a new ISR-owning library
# that nobody added to LIB_ISR) silently dropped it. A dropped handler hangs the chip the
# instant that IRQ fires; this turns a field hang into a red build. CI builds every nrf52840
# target, so this runs on every PR automatically. If a board deliberately stops using one of
# these, edit the tuples on purpose.
_REQUIRED_STRONG = (
"SWI2_EGU2_IRQHandler", # SoftDevice BLE event (SD_EVT) -- advertising & connections
"GPIOTE_IRQHandler", # GPIO interrupts: radio DIO + buttons
"RTC1_IRQHandler", # FreeRTOS scheduler tick
)
# Owned by the TinyUSB stack, so only required when the board builds with USB at all.
# Boards without native USB wiring (e.g. wio-sdk-wm1110's CH340 UART) strip TinyUSB via
# disable_adafruit_usb.py / unflagging USE_TINYUSB, leaving these legitimately weak.
_REQUIRED_STRONG_USB = (
"USBD_IRQHandler", # USB CDC (serial console + 1200bps DFU trigger)
"POWER_CLOCK_IRQHandler", # USB power events (VBUS detect/ready) via TinyUSB hal
)
_tc = env.PioPlatform().get_package_dir("toolchain-gccarmnoneeabi") or ""
_NM = os.path.join(_tc, "bin", "arm-none-eabi-nm")
if not os.path.isfile(_NM):
_NM = "arm-none-eabi-nm" # fall back to PATH
def _assert_isr_handlers_survived(source, target, env):
import subprocess
import sys
try:
# Resolve the ELF at build time; target[0] is the buildprog alias, not the file.
elf = env.subst("$BUILD_DIR/${PROGNAME}.elf")
out = subprocess.check_output([_NM, elf], universal_newlines=True)
except Exception as exc: # tooling hiccup: warn loudly, don't wedge the build
print("nrf52_lto: WARNING - ISR-handler guard skipped (nm failed: %s)" % exc)
return
# nm line: "<addr> <type> <symbol>". type 'T'/'t' = strong (good); 'W'/'w' = weak stub.
kind = {}
for line in out.split("\n"):
f = line.split()
if len(f) >= 3 and f[-1].endswith("_IRQHandler"):
kind[f[-1]] = f[-2]
required = list(_REQUIRED_STRONG)
defines = [
str(d[0] if isinstance(d, tuple) else d) for d in env.get("CPPDEFINES", [])
]
if "USE_TINYUSB" in defines:
required += _REQUIRED_STRONG_USB
dropped = [h for h in required if kind.get(h, "W").upper() != "T"]
if dropped:
sys.stderr.write(
"\n*** nrf52 LTO guard: interrupt handler(s) DROPPED: %s ***\n"
"Each resolved to the weak Default_Handler stub, so the chip hangs when that IRQ\n"
"fires. Compile the .cpp that defines the handler with -fno-lto by adding it to\n"
"LIB_ISR in extra_scripts/nrf52_lto.py. Find the owner of FOO_IRQHandler with:\n"
" grep -rl FOO_IRQHandler <framework-arduinoadafruitnrf52>/{libraries,cores}\n\n"
% ", ".join(dropped)
)
from SCons.Script import Exit
Exit(1) # canonical SCons build-abort -> red build
print(
"nrf52_lto: ISR-handler guard OK -- %d critical handlers strong" % len(required)
)
# Attach to the phony "buildprog" alias, NOT the .elf file node: SCons can skip a post-action
# on a file target during an incremental relink (observed), but the buildprog alias runs every
# build -- so the guard fires on local incremental rebuilds and clean CI builds alike.
env.AddPostAction("buildprog", _assert_isr_handlers_survived)
-140
View File
@@ -1,140 +0,0 @@
#!/usr/bin/env python3
# trunk-ignore-all(ruff/F821)
# trunk-ignore-all(flake8/F821): For SConstruct imports
#
# post:extra_scripts/nrf54l15_linker.py
#
# Fix for Zephyr two-pass link on nRF54L15:
# platformio-build.py registers env.Depends("$PROG_PATH", final_ld_script) but
# the SCons dependency chain is broken (final_ld_script Command never runs).
# This script adds a PreAction on the final firmware binary that runs the gcc
# preprocessing command directly (extracted from build.ninja) to generate
# zephyr/linker.cmd before the link step.
#
# PlatformIO bundles an old Ninja that can't handle multi-output depslog rules,
# so we parse the COMMAND line from build.ninja and run just the gcc -E part,
# skipping the cmake_transform_depfile step (only needed for Ninja deps tracking).
import os
import re
import subprocess
Import("env")
if env.get("PIOENV") != "nrf54l15dk":
pass # Only for the nrf54l15dk environment
else:
def _extract_gcc_command(ninja_build):
"""Parse build.ninja to find the gcc -E command that generates linker.cmd.
The rule format depends on the host:
Windows (CMake's RunCMake wraps every command):
COMMAND = cmd.exe /C "cd /D DIR && arm-none-eabi-gcc.exe ... -o linker.cmd && cmake.exe -E cmake_transform_depfile ..."
POSIX (Linux/macOS — no wrapper):
COMMAND = cd DIR && arm-none-eabi-gcc ... -o linker.cmd && cmake -E cmake_transform_depfile ...
Returns (gcc_cmd_string, cwd_path) or raises RuntimeError.
"""
in_rule = False
with open(ninja_build, "r", encoding="utf-8", errors="replace") as f:
for line in f:
# Detect start of the linker.cmd custom command rule
if not in_rule:
if "build zephyr/linker.cmd" in line and "CUSTOM_COMMAND" in line:
in_rule = True
continue
stripped = line.strip()
if not stripped.startswith("COMMAND = "):
continue
command_val = stripped[len("COMMAND = ") :]
# On Windows the value is wrapped in `cmd.exe /C "..."` — strip
# the wrapper. On POSIX hosts the inner sequence is the value
# itself (no quoting layer).
m = re.search(r'/C\s+"(.*)"\s*$', command_val)
inner = m.group(1) if m else command_val
parts = inner.split(" && ")
cwd = None
gcc_cmd = None
for part in parts:
part = part.strip()
if part.startswith("cd /D "): # Windows form
cwd = part[len("cd /D ") :]
elif part.startswith("cd "): # POSIX form
cwd = part[len("cd ") :]
elif "arm-none-eabi-gcc" in part:
gcc_cmd = part
if not gcc_cmd:
raise RuntimeError(
"nRF54L15 linker fix: arm-none-eabi-gcc command not found in:\n%s"
% inner[:400]
)
return gcc_cmd, cwd
raise RuntimeError(
"nRF54L15 linker fix: 'build zephyr/linker.cmd' rule not found in build.ninja"
)
def _generate_linker_cmd(target, source, env):
"""Generate zephyr/linker.cmd via direct gcc invocation before the final link."""
build_dir = env.subst("$BUILD_DIR")
zephyr_dir = os.path.join(build_dir, "zephyr")
linker_cmd = os.path.join(zephyr_dir, "linker.cmd")
if os.path.exists(linker_cmd):
return # Already present — nothing to do
ninja_build = os.path.join(build_dir, "build.ninja")
if not os.path.exists(ninja_build):
raise RuntimeError(
"nRF54L15 linker fix: build.ninja not found at %s\n"
"Run a full build first so CMake generates the Ninja files."
% ninja_build
)
gcc_cmd, cwd = _extract_gcc_command(ninja_build)
run_cwd = cwd if cwd else zephyr_dir
print(
"==> nRF54L15: Generating zephyr/linker.cmd (LINKER_ZEPHYR_FINAL) via GCC"
)
# gcc_cmd comes verbatim from our own build.ninja (never user input) and
# contains Windows-style paths with spaces that cannot be safely argv-split
# with shlex, so we run it via the platform shell. nosec/nosemgrep below
# acknowledge this deliberate, scoped use of shell=True.
result = subprocess.run( # nosec B602
gcc_cmd,
shell=True, # nosemgrep: python.lang.security.audit.subprocess-shell-true.subprocess-shell-true
cwd=run_cwd,
capture_output=True,
text=True,
)
if result.returncode != 0:
print("GCC stdout:", result.stdout[:2000])
print("GCC stderr:", result.stderr[:2000])
raise RuntimeError(
"nRF54L15 linker fix: GCC failed to generate linker.cmd (rc=%d)"
% result.returncode
)
if not os.path.exists(linker_cmd):
raise RuntimeError(
"nRF54L15 linker fix: GCC returned 0 but linker.cmd was not created at %s"
% linker_cmd
)
print("==> linker.cmd generated successfully")
# Use PIOMAINPROG (set by ZephyrBuildProgram) to get the exact SCons node
prog = env.get("PIOMAINPROG")
if prog:
env.AddPreAction(prog, _generate_linker_cmd)
else:
print(
"[nrf54l15_linker] WARNING: PIOMAINPROG not set, falling back to $PROG_PATH"
)
env.AddPreAction(env.subst("$PROG_PATH"), _generate_linker_cmd)
-7
View File
@@ -33,10 +33,3 @@ tests/reproducers/
# UI-tier camera captures + per-test transcripts. Regenerated every run;
# left on disk for human review between runs.
tests/ui_captures/
# Web stack (FleetSuite): node deps + the built SPA Vite emits into the
# package (regenerated via `cd web-ui && npm run build`). The SQLite registry
# lives under .mtlog/ (already ignored above).
web-ui/node_modules/
web-ui/dist/
src/meshtastic_mcp/web/static/
+17 -37
View File
@@ -174,7 +174,7 @@ rather than auto-`sudo`'ing mid-run.
| `MESHTASTIC_NRFUTIL_BIN` | `$PATH` | Override nrfutil |
| `MESHTASTIC_PICOTOOL_BIN` | `$PATH` | Override picotool |
| `MESHTASTIC_MCP_SEED` | `mcp-<user>-<host>` | PSK seed for test-harness session (CI override) |
| `MESHTASTIC_MCP_FLASH_LOG` | `<mcp-server>/tests/flash.log` | Tee target for pio/esptool/nrfutil subprocess output (web UI tails it) |
| `MESHTASTIC_MCP_FLASH_LOG` | `<mcp-server>/tests/flash.log` | Tee target for pio/esptool/nrfutil subprocess output (TUI tails it) |
| `MESHTASTIC_MCP_TCP_HOST` | unset | `host` or `host:port` of a `meshtasticd` daemon to surface as a TCP device (see "TCP / native-host nodes" below) |
## TCP / native-host nodes
@@ -333,51 +333,31 @@ captures just become 1×1 black PNGs.
**Meshtastic debug** section attached on failure with a 200-line firmware
log tail + device-state dump. Open this first on failures.
- `junit.xml` — CI-parseable.
- `reportlog.jsonl` — `pytest-reportlog` event stream; consumed by the web UI.
- `reportlog.jsonl` — `pytest-reportlog` event stream; consumed by the TUI.
- `fwlog.jsonl` — firmware log mirror (`meshtastic.log.line` pubsub → JSONL).
- `flash.log` — tee of all pio / esptool / nrfutil / picotool subprocess
output during the run (driven by `MESHTASTIC_MCP_FLASH_LOG`).
### Web UI (FleetSuite)
A Vue 3 + Tailwind desktop web app replaces the old Textual TUI. It serves a
FastAPI backend (REST + WebSocket) that reuses the library modules and a
SQLite registry of devices and cameras. See [web-ui/README.md](web-ui/README.md).
**One command** (bootstraps the venv, web deps, npm packages, and SPA build on
first run, then launches):
### Live TUI
```bash
./scripts/fleetsuite.sh # open the native desktop window
./scripts/fleetsuite.sh --browser # serve only → http://127.0.0.1:8765
./scripts/fleetsuite.sh --dev # backend + Vite dev server with hot-reload
./scripts/fleetsuite.sh --rebuild # force a fresh SPA build first
.venv/bin/meshtastic-mcp-test-tui
.venv/bin/meshtastic-mcp-test-tui tests/mesh # pytest args pass through
```
In **VS Code**: Run Task → **FleetSuite** (or _FleetSuite: Dev (HMR)_). Or do it
by hand:
Textual-based wrapper over `run-tests.sh` with a live test tree, tier
counters, pytest output pane, firmware-log pane, and a device-status strip.
Key bindings: `r` re-run focused, `f` filter, `d` failure detail, `g` open
`report.html`, `x` export reproducer bundle, `l` cycle fw-log filter, `q`
quit (SIGINT → SIGTERM → SIGKILL escalation).
```bash
.venv/bin/pip install -e '.[web]' # FastAPI + uvicorn + aiosqlite + pywebview
(cd web-ui && npm install && npm run build) # build the SPA into web/static
.venv/bin/meshtastic-mcp-web # opens a native pywebview window
.venv/bin/meshtastic-mcp-web --browser # serve only → http://127.0.0.1:8765
```
Two views: **Fleet** (per-device cards keyed by USB serial that follow a device
across changing ports, with live serial logs, packet/telemetry, test history,
full device control, and an assignable live USB camera feed) and **Test Suite**
(run/stop `run-tests.sh`, live tier counters + test tree, pytest/flash/firmware
log panes, run history). The header shows the firmware branch/SHA/dirty live.
**Native nodes (Docker).** The Fleet view can run Linux-native `meshtasticd`
nodes in Docker (no radio, sim mode) and manage them as TCP devices — run /
stop / restart / remove, live `docker logs`, config/send-text/inject-NodeDB over
`tcp://`. By default it builds the checkout's own image from `Dockerfile`; set
`MESHTASTIC_NATIVE_IMAGE` (e.g. `meshtastic/meshtasticd:latest`) to use a
prebuilt image instead.
For development with HMR, run `scripts/web-dev.sh` (backend + Vite together).
Set `MESHTASTIC_UI_TUI_CAMERA=1` to mount a bottom-of-screen **UI camera**
panel. Left side: the latest capture PNG rendered as Unicode half-blocks
(via `rich-pixels`, works in any terminal — no kitty/sixel required).
Right side: live transcript tail ("step 3 — frame 4/8 name=nodelist_nodes
— OCR: Nodes 2/2") so you can see every event-injection and its result
as each UI test runs. Requires the `[ui]` extras for image rendering; the
transcript alone works without them.
### Slash commands
+13 -14
View File
@@ -17,16 +17,11 @@ test = [
"pytest-timeout>=2.3",
"coverage[toml]>=7",
"pyyaml>=6",
]
# Web stack (FastAPI backend + pywebview desktop window) that replaces the
# Textual TUI. Serves the built Vue SPA in `web/static`. Camera streaming
# reuses the `[ui]` extra (opencv-python-headless).
web = [
"fastapi>=0.110",
"uvicorn[standard]>=0.27",
"aiosqlite>=0.19",
"pywebview>=5.0",
"requests>=2.31",
# textual is required by the `meshtastic-mcp-test-tui` script (see
# `src/meshtastic_mcp/cli/test_tui.py`). Bundled into `test` rather than a
# separate `[tui]` extra because v1 expects test operators are the only
# consumers; revisit if install cost pushes back.
"textual>=0.50",
]
# UI test tier + `capture_screen` MCP tool. Optional because the ML OCR
# model alone is ~100 MB and camera hardware is user-supplied.
@@ -37,15 +32,19 @@ ui = [
"numpy>=1.26",
"easyocr>=1.7",
"Pillow>=10.0",
# Renders the latest camera capture as Unicode half-blocks in the TUI
# (MESHTASTIC_UI_TUI_CAMERA=1). Terminal-agnostic — no kitty / sixel
# dependency. Pure Python, tiny.
"rich-pixels>=3.0",
]
ui-min = ["opencv-python-headless>=4.9", "numpy>=1.26"]
[project.scripts]
meshtastic-mcp = "meshtastic_mcp.__main__:main"
# Web stack replacing the Textual TUI: a FastAPI backend (REST + WebSocket)
# that reuses the library modules and serves the Vue SPA, opened in a
# pywebview window (`--browser` to serve only).
meshtastic-mcp-web = "meshtastic_mcp.web.__main__:main"
# Live TUI wrapping run-tests.sh — shells out to the same script the plain
# CLI uses, tails pytest-reportlog for per-test state, and polls the device
# list at startup + post-run (port lock forces it to stay idle during the run).
meshtastic-mcp-test-tui = "meshtastic_mcp.cli.test_tui:main"
[build-system]
requires = ["hatchling"]
+2 -6
View File
@@ -191,12 +191,10 @@ echo
# PASS/FAIL — every hardware test would SKIP with "role not present". We
# narrow to tests/unit explicitly so the summary reads as "no hardware,
# unit suite only" instead of "big skip count looks suspicious".
# Keep terminal output condensed (`-q -r fE`) so skip-heavy runs do not print
# each skipped test in full; skip counts still appear in pytest's summary.
if [[ -z $DETECTED && $# -eq 0 ]]; then
echo "[pre-flight] no supported devices detected; running unit tier only."
echo
exec "$VENV_PY" -m pytest tests/unit -q -r fE --report-log=tests/reportlog.jsonl
exec "$VENV_PY" -m pytest tests/unit -v --report-log=tests/reportlog.jsonl
fi
# Default pytest args when the user passed none. Power users can invoke
@@ -212,13 +210,11 @@ fi
# skipping half the hardware tests with "not baked with session profile"
# errors. Power users who know their hardware is current and want to shave
# those seconds can pass `--assume-baked` explicitly.
# Defaults also use condensed reporting (`-q -r fE`) to avoid listing every
# skipped test verbatim while still surfacing failures/errors and summary data.
if [[ $# -eq 0 ]]; then
set -- tests/ \
--html=tests/report.html --self-contained-html \
--junitxml=tests/junit.xml \
-q -r fE --tb=short
-v --tb=short
fi
# UI tier requires opencv-python-headless (and ideally easyocr). If it's
-77
View File
@@ -1,77 +0,0 @@
#!/usr/bin/env bash
# Single entrypoint for FleetSuite — the web UI for the Meshtastic test harness.
# Ensures deps, builds the SPA, and launches the app. One command, from anywhere.
#
# ./scripts/fleetsuite.sh # build SPA if needed, open the desktop window
# ./scripts/fleetsuite.sh --browser # serve only → http://127.0.0.1:8765
# ./scripts/fleetsuite.sh --dev # backend + Vite dev server with hot-reload
# ./scripts/fleetsuite.sh --rebuild # force a fresh SPA build first
#
# First run bootstraps the venv + web deps + npm packages automatically.
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT"
DEV=0
BROWSER=0
REBUILD=0
for arg in "$@"; do
case "$arg" in
--dev) DEV=1 ;;
--browser) BROWSER=1 ;;
--rebuild) REBUILD=1 ;;
-h | --help)
sed -n '2,11p' "${BASH_SOURCE[0]}"
exit 0
;;
*)
echo "unknown arg: $arg (try --help)" >&2
exit 2
;;
esac
done
PY="$ROOT/.venv/bin/python"
STATIC="$ROOT/src/meshtastic_mcp/web/static/index.html"
note() { printf '\033[36m[fleetsuite]\033[0m %s\n' "$*"; }
# 1. Python venv + web extra ------------------------------------------------
if [[ ! -x $PY ]]; then
note "creating venv (.venv)…"
python3 -m venv "$ROOT/.venv"
fi
if ! "$PY" -c 'import fastapi, aiosqlite, uvicorn, webview' >/dev/null 2>&1; then
note "installing the [web] extra…"
"$PY" -m pip install --quiet --upgrade pip
"$PY" -m pip install --quiet -e "$ROOT[web]"
fi
# 2. Dev mode: backend + Vite with HMR --------------------------------------
if [[ $DEV == 1 ]]; then
note "dev mode (backend :8765 + Vite HMR)"
exec "$ROOT/scripts/web-dev.sh"
fi
# 3. Frontend deps + production build ----------------------------------------
if ! command -v npm >/dev/null 2>&1; then
echo "npm not found — install Node.js (https://nodejs.org) to build the UI." >&2
exit 1
fi
if [[ ! -d "$ROOT/web-ui/node_modules" ]]; then
note "installing web-ui npm packages…"
(cd "$ROOT/web-ui" && npm install)
fi
if [[ $REBUILD == 1 || ! -f $STATIC ]]; then
note "building the SPA…"
(cd "$ROOT/web-ui" && npm run build)
fi
# 4. Launch ------------------------------------------------------------------
if [[ $BROWSER == 1 ]]; then
note "serving at http://127.0.0.1:8765 (Ctrl-C to stop)"
exec "$ROOT/.venv/bin/meshtastic-mcp-web" --browser
fi
note "opening FleetSuite window…"
exec "$ROOT/.venv/bin/meshtastic-mcp-web"
-32
View File
@@ -1,32 +0,0 @@
#!/usr/bin/env bash
# Run the FleetSuite backend (uvicorn, :8765) and the Vite dev server together,
# with HMR. The Vite server proxies /api and /ws to the backend. Ctrl-C stops
# both. For a single-process production run instead, build the SPA once
# (`cd web-ui && npm run build`) and launch `meshtastic-mcp-web`.
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT"
PY="${ROOT}/.venv/bin/python"
if [[ ! -x $PY ]]; then
echo "No venv at .venv — run: python3 -m venv .venv && .venv/bin/pip install -e '.[web,test]'" >&2
exit 1
fi
cleanup() {
trap - INT TERM
[[ -n ${BACK_PID-} ]] && kill "$BACK_PID" 2>/dev/null || true
[[ -n ${UI_PID-} ]] && kill "$UI_PID" 2>/dev/null || true
}
trap cleanup INT TERM EXIT
echo "[web-dev] starting backend on http://127.0.0.1:8765 …"
"$PY" -m uvicorn meshtastic_mcp.web.app:create_app --factory --port 8765 --reload &
BACK_PID=$!
echo "[web-dev] starting Vite dev server …"
(cd web-ui && npm run dev) &
UI_PID=$!
wait
File diff suppressed because it is too large Load Diff
-382
View File
@@ -1,382 +0,0 @@
"""Fake NodeDB fixture push — Portduino file copy + hardware XModem upload.
The fixture pipeline is two-stage:
1. `bin/gen-fake-nodedb-seed.py` produces a deterministic JSONL describing N
fake-but-realistic peers. Committed under `test/fixtures/nodedb/`.
2. `bin/seed-json-to-proto.py` compiles JSONL → binary v25 NodeDatabase
protobuf with fresh wall-clock timestamps.
This module exposes `push_fake_nodedb(...)`, the MCP tool that:
- target="portduino": compiles the JSONL into the device's prefs dir on
the local filesystem (`~/.portduino/<config>/prefs/nodes.proto`).
- target="hardware": compiles to a temp file, then streams it over the
XModem protocol (via the meshtastic SerialInterface/BLEInterface +
`meshtastic.xmodempacket` pubsub topic) to `/prefs/nodes.proto` on the
device. Triggers a reboot so the firmware loads the new state on next
boot.
XModem wire details (mirrors firmware impl at src/xmodem.cpp:115-260):
* 128-byte chunks; final chunk padded to 128 B with 0x1A (SUB) bytes.
* CRC16-CCITT (poly 0x1021, init 0x0000).
* SOH/seq=0 carries the destination filename in `buffer.bytes`. ACK if
`FSCom.open(filename, FILE_O_WRITE)` succeeds; NAK otherwise.
* SOH/seq≥1 carries a 128-byte chunk. ACK = advance; NAK = retransmit.
* EOT after the last chunk flushes + closes the file on-device.
Hardware push requires `confirm=True` (mirrors factory_reset / erase_and_flash
in the .github/copilot-instructions.md "never do these without asking" list).
"""
from __future__ import annotations
import dataclasses
import hashlib
import pathlib
import queue
import shutil
import subprocess
import sys
import tempfile
import time
from typing import Any, Literal
from .connection import connect, is_tcp_port
# Resolve repo root so the tool works regardless of mcp-server cwd.
_REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]
_SEED_DIR = _REPO_ROOT / "test" / "fixtures" / "nodedb"
_COMPILE_SCRIPT = _REPO_ROOT / "bin" / "seed-json-to-proto.py"
_DEFAULT_NODES_FILENAME = "/prefs/nodes.proto"
_XMODEM_CHUNK = 128
_XMODEM_SUB = 0x1A
_ACK_TIMEOUT_INIT_S = 5.0
_ACK_TIMEOUT_CHUNK_S = 2.0
_MAX_CHUNK_RETRIES = 5
_VALID_SIZES = (250, 500, 1000, 2000)
class FixtureError(RuntimeError):
"""Raised for any fixture-push failure (compile, transport, ack timeout, …)."""
# ---------------------------------------------------------------------------
# CRC16-CCITT (poly 0x1021, init 0x0000). Matches the firmware's `crc16_ccitt`.
# Hand-rolled to avoid the optional `crcmod` dep.
# ---------------------------------------------------------------------------
def _crc16_ccitt(data: bytes, *, init: int = 0x0000) -> int:
crc = init
for b in data:
crc ^= b << 8
for _ in range(8):
if crc & 0x8000:
crc = ((crc << 1) ^ 0x1021) & 0xFFFF
else:
crc = (crc << 1) & 0xFFFF
return crc
# ---------------------------------------------------------------------------
# Compile step — shells out to bin/seed-json-to-proto.py so the MCP module
# doesn't have to duplicate the proto-encoding logic.
# ---------------------------------------------------------------------------
def _compile_proto(jsonl_path: pathlib.Path, out_path: pathlib.Path) -> None:
if not _COMPILE_SCRIPT.is_file():
raise FixtureError(f"compile script missing at {_COMPILE_SCRIPT}")
cmd = [
sys.executable,
str(_COMPILE_SCRIPT),
"--in",
str(jsonl_path),
"--out",
str(out_path),
]
try:
subprocess.run(cmd, check=True, capture_output=True, text=True)
except subprocess.CalledProcessError as exc:
raise FixtureError(
f"seed-json-to-proto.py failed (exit {exc.returncode}):\n"
f" stdout: {exc.stdout}\n stderr: {exc.stderr}"
) from exc
def _resolve_seed_jsonl(size: int, custom: str | None) -> pathlib.Path:
if custom is not None:
p = pathlib.Path(custom).expanduser().resolve()
if not p.is_file():
raise FixtureError(f"custom_seed_jsonl not found: {p}")
return p
p = _SEED_DIR / f"seed_v25_{size:04d}.jsonl"
if not p.is_file():
raise FixtureError(
f"missing committed seed at {p}. "
f"Run `./bin/regen-fake-nodedbs.sh` to generate it."
)
return p
# ---------------------------------------------------------------------------
# Portduino push — file copy into ~/.portduino/<config>/prefs/
# ---------------------------------------------------------------------------
def _portduino_prefs_dir(config_name: str) -> pathlib.Path:
home = pathlib.Path.home()
return home / ".portduino" / config_name / "prefs"
def _push_portduino(
size: int,
jsonl: pathlib.Path,
portduino_config: str,
backup_existing: bool,
) -> dict[str, Any]:
prefs = _portduino_prefs_dir(portduino_config)
prefs.mkdir(parents=True, exist_ok=True)
target = prefs / "nodes.proto"
backed_up_to: str | None = None
if backup_existing and target.is_file():
ts = int(time.time())
backup = prefs / f"nodes.proto.bak.{ts}"
shutil.move(str(target), str(backup))
backed_up_to = str(backup)
_compile_proto(jsonl, target)
raw = target.read_bytes()
return {
"transport": "portduino",
"path": str(target),
"bytes": len(raw),
"sha256": hashlib.sha256(raw).hexdigest(),
"jsonl_source": str(jsonl),
"backed_up_to": backed_up_to,
}
# ---------------------------------------------------------------------------
# Hardware push — XModem over BLE/serial via the meshtastic Python interface.
# ---------------------------------------------------------------------------
@dataclasses.dataclass
class _AckEvent:
control: int
seq: int
def _wait_for_response(q: "queue.Queue[_AckEvent]", timeout_s: float) -> _AckEvent:
try:
return q.get(timeout=timeout_s)
except queue.Empty as exc:
raise FixtureError(
f"XModem response timeout after {timeout_s:.1f}s — device not responding"
) from exc
def _push_hardware(
size: int,
jsonl: pathlib.Path,
port: str | None,
reboot_after: bool,
) -> dict[str, Any]:
# Lazy imports so the module loads even when the meshtastic deps aren't
# available (e.g. CI in a Python env without the package installed).
try:
from meshtastic.protobuf import mesh_pb2, xmodem_pb2
from pubsub import pub
except ImportError as exc: # pragma: no cover — dep missing
raise FixtureError(
f"hardware push requires the meshtastic + pypubsub packages: {exc}"
) from exc
if is_tcp_port(port):
raise FixtureError(
"hardware push over TCP/portduino is not supported — use "
"target='portduino' to drop the fixture directly into the prefs dir."
)
# Compile the fixture to a temp file with fresh timestamps.
with tempfile.NamedTemporaryFile(suffix=".proto", delete=False) as tf:
proto_path = pathlib.Path(tf.name)
try:
_compile_proto(jsonl, proto_path)
payload = proto_path.read_bytes()
finally:
proto_path.unlink(missing_ok=True)
sha256 = hashlib.sha256(payload).hexdigest()
total_bytes = len(payload)
# Subscribe to XModem responses BEFORE we open the interface, so we don't
# race the first ACK that arrives during the SOH/seq=0 handshake.
#
# NB: the signature MUST declare every kwarg pypubsub will see for this
# topic, or pubsub locks the topic spec to a smaller set (whichever
# subscribe arrives first) and then *rejects* the meshtastic library's
# publish call with `SenderUnknownMsgDataError: unknown ... interface`.
# The meshtastic lib publishes both `packet=` and `interface=`
# (mesh_interface.py:1389-1395), so both must appear here.
response_q: "queue.Queue[_AckEvent]" = queue.Queue()
def _on_xmodem(packet: Any = None, interface: Any = None, **_kw: Any) -> None:
if packet is None:
return
response_q.put(_AckEvent(control=int(packet.control), seq=int(packet.seq)))
pub.subscribe(_on_xmodem, "meshtastic.xmodempacket")
chunks_sent = 0
retried = 0
rebooted = False
XMC = xmodem_pb2.XModem.Control
try:
with connect(port=port) as iface:
# 1) Send the filename (SOH, seq=0).
init_pkt = xmodem_pb2.XModem(
control=XMC.Value("SOH"),
seq=0,
buffer=_DEFAULT_NODES_FILENAME.encode("utf-8"),
)
iface._sendToRadio(mesh_pb2.ToRadio(xmodemPacket=init_pkt))
ack = _wait_for_response(response_q, _ACK_TIMEOUT_INIT_S)
if ack.control != XMC.Value("ACK"):
raise FixtureError(
f"device refused filename {_DEFAULT_NODES_FILENAME!r} "
f"(got control={ack.control}, expected ACK). "
f"Filesystem full or permissions issue?"
)
# 2) Stream the payload in 128 B chunks.
for offset in range(0, total_bytes, _XMODEM_CHUNK):
chunk = payload[offset : offset + _XMODEM_CHUNK]
if len(chunk) < _XMODEM_CHUNK:
# Pad final chunk to 128 B with SUB. The trailing 0x1A bytes
# become part of the file on-device, but nanopb ignores
# bytes past the end of the top-level message.
chunk = chunk + bytes([_XMODEM_SUB] * (_XMODEM_CHUNK - len(chunk)))
seq = ((offset // _XMODEM_CHUNK) + 1) % 256
# Retry loop on NAK / timeout.
attempts = 0
while True:
pkt = xmodem_pb2.XModem(
control=XMC.Value("SOH"),
seq=seq,
buffer=chunk,
crc16=_crc16_ccitt(chunk),
)
iface._sendToRadio(mesh_pb2.ToRadio(xmodemPacket=pkt))
ack = _wait_for_response(response_q, _ACK_TIMEOUT_CHUNK_S)
if ack.control == XMC.Value("ACK"):
chunks_sent += 1
break
if ack.control == XMC.Value("NAK"):
attempts += 1
retried += 1
if attempts >= _MAX_CHUNK_RETRIES:
# Abort: send CAN so the firmware removes the half-
# written file via FSCom.remove(filename).
iface._sendToRadio(
mesh_pb2.ToRadio(
xmodemPacket=xmodem_pb2.XModem(
control=XMC.Value("CAN")
)
)
)
raise FixtureError(
f"chunk seq={seq} NAK'd {attempts} times; "
f"aborted transfer (file removed on-device)."
)
continue # retry the same chunk
raise FixtureError(
f"unexpected XModem control={ack.control} on seq={seq}"
)
# 3) Tell the device we're done.
iface._sendToRadio(
mesh_pb2.ToRadio(
xmodemPacket=xmodem_pb2.XModem(control=XMC.Value("EOT"))
)
)
ack = _wait_for_response(response_q, _ACK_TIMEOUT_CHUNK_S)
if ack.control != XMC.Value("ACK"):
raise FixtureError(f"EOT not ACKed (got control={ack.control})")
# 4) Reboot so loadFromDisk picks up the new file.
if reboot_after:
iface.localNode.reboot(secs=1)
rebooted = True
finally:
try:
pub.unsubscribe(_on_xmodem, "meshtastic.xmodempacket")
except Exception:
pass
return {
"transport": "hardware",
"port": port,
"filename_on_device": _DEFAULT_NODES_FILENAME,
"bytes": total_bytes,
"chunks_sent": chunks_sent,
"retried": retried,
"sha256": sha256,
"jsonl_source": str(jsonl),
"rebooted": rebooted,
}
# ---------------------------------------------------------------------------
# Public entry point — registered as an MCP tool in server.py.
# ---------------------------------------------------------------------------
def push_fake_nodedb(
size: int,
target: Literal["portduino", "hardware"] = "portduino",
*,
port: str | None = None,
portduino_config: str = "default",
backup_existing: bool = True,
confirm: bool = False,
reboot_after: bool = True,
custom_seed_jsonl: str | None = None,
) -> dict[str, Any]:
"""Compile a fresh-timestamp NodeDatabase fixture and push it to a device.
Args:
size: 250, 500, 1000, or 2000 — selects which committed seed JSONL to use.
target: "portduino" (file copy to ~/.portduino/<config>/prefs/) or
"hardware" (XModem upload to /prefs/nodes.proto + reboot).
port: required for target="hardware". Serial path (e.g. /dev/cu.usbmodemXXXX)
or BLE identifier. TCP endpoints are rejected — use target="portduino"
instead.
portduino_config: which Portduino instance dir under ~/.portduino/. Default "default".
backup_existing: portduino only. Move nodes.proto -> nodes.proto.bak.<ts>
if present, so you can roll back.
confirm: required True for target="hardware" (writes flash + reboots).
reboot_after: hardware only. If True, send a 1-second reboot after the
final ACK so loadFromDisk picks up the new file at next boot.
custom_seed_jsonl: override the committed JSONL. Use to push a hand-edited
test scenario.
Returns:
dict with transport, bytes, sha256, etc. — depends on target.
"""
if size not in _VALID_SIZES:
raise FixtureError(
f"size must be one of {_VALID_SIZES}; got {size!r}. "
f"Add a new committed seed if you need a different cardinality."
)
jsonl = _resolve_seed_jsonl(size, custom_seed_jsonl)
if target == "portduino":
return _push_portduino(size, jsonl, portduino_config, backup_existing)
if target == "hardware":
if not confirm:
raise FixtureError(
"hardware push writes flash and triggers a reboot — pass confirm=True."
)
if not port:
raise FixtureError(
"target='hardware' requires a port (e.g. /dev/cu.usbmodemXXXX)."
)
return _push_hardware(size, jsonl, port, reboot_after)
raise FixtureError(f"unknown target {target!r}; expected 'portduino' or 'hardware'")
-43
View File
@@ -15,7 +15,6 @@ from . import (
admin,
boards,
devices,
fixtures,
flash,
hw_tools,
info,
@@ -964,45 +963,3 @@ def recorder_export(
dest_dir=dest_dir,
streams=streams,
)
# ---------- Fixture / test-data push --------------------------------------
@app.tool()
def push_fake_nodedb(
size: int,
target: str = "portduino",
port: str | None = None,
portduino_config: str = "default",
backup_existing: bool = True,
confirm: bool = False,
reboot_after: bool = True,
custom_seed_jsonl: str | None = None,
) -> dict[str, Any]:
"""Push a fake-NodeDB v25 fixture (250/500/1000/2000 nodes) onto a device.
Two transports:
target="portduino" — file copy to ~/.portduino/<portduino_config>/prefs/nodes.proto.
Fast, no device connection needed.
target="hardware" — XModem upload over serial/BLE to /prefs/nodes.proto.
Requires `port` + `confirm=True`. Triggers a reboot
so loadFromDisk picks up the new file at next boot.
Compiles a fresh-timestamp proto from the committed JSONL seed under
test/fixtures/nodedb/seed_v25_<N>.jsonl each invocation, so the loaded
NodeDB always looks "recent" to the connecting phone. Structural data
(names, IDs, positions, telemetries) is deterministic per the seed.
Override the JSONL via `custom_seed_jsonl` to push a hand-edited scenario.
"""
return fixtures.push_fake_nodedb(
size=size,
target=target, # type: ignore[arg-type]
port=port,
portduino_config=portduino_config,
backup_existing=backup_existing,
confirm=confirm,
reboot_after=reboot_after,
custom_seed_jsonl=custom_seed_jsonl,
)
@@ -1,11 +0,0 @@
"""FleetSuite — the FastAPI + WebSocket backend that serves the Vue SPA and
drives the Meshtastic test harness. Replaces the old Textual TUI.
Layout:
db/ — aiosqlite registry (devices, cameras, flash, runs, builds, settings)
services/ — identity reconciliation, control gating, firmware builds, the
pytest runner, the Datadog forwarder
ws/ — the single broadcast hub backing ``/ws``
app.py — ``create_app()`` factory (REST + ``/ws``, serves ``web/static``)
__main__.py — ``main()``: serve + open a pywebview window (``--browser`` to skip it)
"""
@@ -1,79 +0,0 @@
"""FleetSuite entrypoint (the ``meshtastic-mcp-web`` console script).
Default: serve the API + built SPA on 127.0.0.1:8765 and open a pywebview
desktop window pointed at it. ``--browser`` serves only (no window) so you can
open it in any browser — also the mode used in headless/CI and by agents.
"""
from __future__ import annotations
import argparse
import logging
import threading
import time
import uvicorn
HOST = "127.0.0.1"
PORT = 8765
def _serve(server: uvicorn.Server) -> None:
server.run()
def main() -> None:
parser = argparse.ArgumentParser(prog="meshtastic-mcp-web", description=__doc__)
parser.add_argument(
"--browser",
action="store_true",
help="serve only (no desktop window) — open http://127.0.0.1:8765 yourself",
)
parser.add_argument("--host", default=HOST)
parser.add_argument("--port", type=int, default=PORT)
args = parser.parse_args()
logging.basicConfig(
level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s"
)
config = uvicorn.Config(
"meshtastic_mcp.web.app:create_app",
factory=True,
host=args.host,
port=args.port,
log_level="info",
)
server = uvicorn.Server(config)
if args.browser:
server.run()
return
# Desktop window: run uvicorn on a background thread, open pywebview on the
# main thread (some platforms require the GUI loop to own the main thread).
try:
import webview # type: ignore
except Exception: # noqa: BLE001
logging.warning("pywebview unavailable — falling back to --browser mode")
server.run()
return
thread = threading.Thread(target=_serve, args=(server,), daemon=True)
thread.start()
# Wait for the server to bind before pointing the window at it.
deadline = time.monotonic() + 15
while not server.started and time.monotonic() < deadline:
time.sleep(0.1)
webview.create_window(
"FleetSuite", f"http://{args.host}:{args.port}", width=1400, height=900
)
webview.start()
server.should_exit = True
thread.join(timeout=5)
if __name__ == "__main__":
main()
-611
View File
@@ -1,611 +0,0 @@
"""FastAPI application factory for FleetSuite.
``create_app()`` wires the registry, the broadcast hub, and the services
together in a lifespan, mounts the REST API + the single ``/ws`` socket, and
serves the built Vue SPA from ``web/static``. Blocking library calls (serial
I/O, pio, git) are dispatched to a thread so the event loop stays responsive.
"""
from __future__ import annotations
import asyncio
import logging
from pathlib import Path
from fastapi import APIRouter, Body, FastAPI, HTTPException, Request, WebSocket
from fastapi.responses import JSONResponse, StreamingResponse
from fastapi.staticfiles import StaticFiles
from meshtastic_mcp import (
admin,
boards,
fixtures,
flash as flash_lib,
info as mt_info,
log_query,
)
from .db import repo_builds as rb
from .db import repo_cameras as rc
from .db import repo_devices as rd
from .db import repo_flash as rf
from .db import repo_runs as rr
from .db.database import Database, default_db_path
from .services import (
builder,
camera_stream,
control,
datadog,
discovery,
firmware,
identity,
native,
power,
serial_monitor,
test_runner,
)
from .services.control import ControlBusy
from .services.power import AmbiguousPort, NoPort
from .ws.hub import Connection, Hub
log = logging.getLogger("meshtastic_mcp.web")
STATIC_DIR = Path(__file__).parent / "static"
def _busy_guard(exc: ControlBusy) -> HTTPException:
return HTTPException(status_code=409, detail=str(exc))
def create_app() -> FastAPI:
app = FastAPI(title="FleetSuite", version="0.1.0")
# --- lifespan: own the db + services for the process lifetime ----------
@app.on_event("startup")
async def _startup() -> None:
db = await Database(default_db_path()).connect()
hub = Hub()
hub.bind_loop(asyncio.get_running_loop())
app.state.db = db
app.state.hub = hub
app.state.orch = builder.BuildOrchestrator(db, hub)
app.state.runner = test_runner.TestRunner(db, hub)
app.state.forwarder = datadog.DDForwarder(db, hub)
await app.state.forwarder.reload()
app.state.serialmon = serial_monitor.SerialMonitor(db, hub)
# Discovery auto-enriches devices, suspending their serial monitor for
# the connect — so it needs the monitor handle.
app.state.discovery = discovery.DeviceDiscovery(
db, hub, serialmon=app.state.serialmon
)
app.state.discovery.start()
log.info("FleetSuite started — registry at %s", db.path)
@app.on_event("shutdown")
async def _shutdown() -> None:
disc = getattr(app.state, "discovery", None)
if disc:
await disc.stop()
sm = getattr(app.state, "serialmon", None)
if sm:
await sm.shutdown()
db = getattr(app.state, "db", None)
if db:
await db.close()
api = APIRouter(prefix="/api")
_mount_devices(api)
_mount_cameras(api)
_mount_firmware(api)
_mount_builds(api)
_mount_datadog(api)
_mount_tests(api)
_mount_native(api)
_mount_boards(api)
_mount_hubs(api)
app.include_router(api)
_mount_ws(app)
@app.exception_handler(ControlBusy)
async def _busy(_req: Request, exc: ControlBusy):
return JSONResponse(status_code=409, content={"detail": str(exc)})
if STATIC_DIR.is_dir():
app.mount("/", StaticFiles(directory=str(STATIC_DIR), html=True), name="spa")
else:
log.warning("no built SPA at %s — run `npm run build` in web-ui/", STATIC_DIR)
return app
# --- helpers ---------------------------------------------------------------
async def _device_or_404(db: Database, serial: str) -> dict:
row = await rd.get(db, serial)
if row is None:
raise HTTPException(status_code=404, detail=f"unknown device: {serial}")
return row
def _gate_idle() -> None:
try:
control._ensure_idle()
except ControlBusy as exc:
raise _busy_guard(exc)
async def _port_action(request: Request, serial: str, fn, *args):
"""Run a blocking port-bound library call, suspending any live serial
monitor for the device so the USB port is free, then resuming it."""
_gate_idle()
sm = request.app.state.serialmon
await sm.suspend(serial)
try:
return await asyncio.to_thread(fn, *args)
finally:
await sm.resume(serial)
# --- devices ---------------------------------------------------------------
def _mount_devices(api: APIRouter) -> None:
@api.get("/devices")
async def list_devices(request: Request):
return await rd.list_all(request.app.state.db)
@api.patch("/devices/{serial}")
async def patch_device(serial: str, request: Request, body: dict = Body(...)):
db, hub = request.app.state.db, request.app.state.hub
await _device_or_404(db, serial)
if "friendly_name" in body:
dev = await rd.set_friendly_name(db, serial, body["friendly_name"])
else:
dev = await rd.get(db, serial)
await hub.publish("device.update", dev)
return dev
@api.put("/devices/{serial}/env")
async def set_env(serial: str, request: Request, body: dict = Body(...)):
db, hub = request.app.state.db, request.app.state.hub
await _device_or_404(db, serial)
env = body.get("env")
# A provided env pins it; clearing it releases the pin to auto-detect.
dev = await rd.set_env(db, serial, env, locked=env is not None)
await hub.publish("device.update", dev)
return dev
@api.post("/devices/{serial}/refresh")
async def refresh(serial: str, request: Request):
db, hub = request.app.state.db, request.app.state.hub
row = await _device_or_404(db, serial)
port = row.get("current_port")
info = await _port_action(request, serial, mt_info.device_info, port)
hw_model = info.get("hw_model")
env = identity.env_for_hw_model(hw_model) if hw_model else None
dev = await rd.update_enrichment(
db,
serial,
node_num=info.get("my_node_num"),
env=env,
hw_model=hw_model,
firmware_version=info.get("firmware_version"),
region=info.get("region"),
)
await hub.publish("device.update", dev)
return {"device": dev}
@api.get("/devices/{serial}/flash-stats")
async def flash_stats(serial: str, request: Request):
return await rf.comparison(request.app.state.db, serial)
@api.post("/devices/{serial}/flash")
async def flash_device(serial: str, request: Request):
db, hub = request.app.state.db, request.app.state.hub
row = await _device_or_404(db, serial)
env = control.env_for_device(row)
port = row.get("current_port")
if not env or not port:
raise HTTPException(status_code=400, detail="no env/port resolved")
loop = asyncio.get_running_loop()
start = loop.time()
result = await _port_action(
request, serial, lambda: flash_lib.flash(env, port, confirm=True)
)
duration = round(loop.time() - start, 2)
ok = result.get("exit_code") == 0
fw = firmware.firmware_ref()
await rf.record(
db,
device_serial=serial,
env=env,
fw_sha=fw.get("sha"),
from_artifact=False,
duration_s=duration,
ok=ok,
)
if ok:
await rd.record_flashed(db, serial, branch=fw.get("branch"), sha=fw.get("sha"))
await hub.publish("device.update", await rd.get(db, serial))
return {"ok": ok, "duration_s": duration, **result}
@api.post("/devices/{serial}/reboot")
async def reboot(serial: str, request: Request):
row = await _device_or_404(request.app.state.db, serial)
return await _port_action(request, serial, admin.reboot, row.get("current_port"), True, 5)
@api.post("/devices/{serial}/factory-reset")
async def factory_reset(serial: str, request: Request):
row = await _device_or_404(request.app.state.db, serial)
return await _port_action(
request, serial, admin.factory_reset, row.get("current_port"), True
)
@api.post("/devices/{serial}/send-text")
async def send_text(serial: str, request: Request, body: dict = Body(...)):
row = await _device_or_404(request.app.state.db, serial)
text = body.get("text", "")
return await _port_action(
request, serial, admin.send_text, text, None, 0, False, row.get("current_port")
)
@api.post("/devices/{serial}/inject-nodedb")
async def inject_nodedb(serial: str, request: Request, body: dict = Body(...)):
row = await _device_or_404(request.app.state.db, serial)
size = int(body.get("size", 500))
return await _port_action(request, serial, _inject, size, row.get("current_port"))
@api.get("/devices/{serial}/config")
async def get_config(serial: str, request: Request, section: str | None = None):
row = await _device_or_404(request.app.state.db, serial)
return await _port_action(
request, serial, admin.get_config, section, row.get("current_port")
)
@api.put("/devices/{serial}/config")
async def set_config(serial: str, request: Request, body: dict = Body(...)):
row = await _device_or_404(request.app.state.db, serial)
path = body.get("path")
if not path:
raise HTTPException(status_code=400, detail="missing config path")
return await _port_action(
request, serial, admin.set_config, path, body.get("value"), row.get("current_port")
)
@api.get("/devices/{serial}/packets")
async def device_packets(
serial: str, request: Request, start: str = "-30m", max: int = 100
):
await _device_or_404(request.app.state.db, serial)
# Recorder packets are mesh-wide, not keyed by USB port — return the
# recent window so the per-device tab has live traffic to show.
window = await asyncio.to_thread(
lambda: log_query.packets_window(start, "now", max=max)
)
return {"packets": window.get("packets", [])}
@api.get("/devices/{serial}/test-results")
async def device_test_results(serial: str, request: Request, limit: int = 100):
rows = await rr.results_for_device(request.app.state.db, serial)
return rows[:limit]
# --- per-device USB power (uhubctl) ----------------------------------
@api.put("/devices/{serial}/hub-port")
async def set_hub_port(serial: str, request: Request, body: dict = Body(...)):
db, hub = request.app.state.db, request.app.state.hub
await _device_or_404(db, serial)
loc = body.get("location")
port = body.get("port")
dev = await rd.set_hub_port(
db, serial, location=loc, port=int(port) if port is not None else None
)
await hub.publish("device.update", dev)
return dev
@api.post("/devices/{serial}/locate")
async def locate_device(serial: str, request: Request):
db, hub = request.app.state.db, request.app.state.hub
await _device_or_404(db, serial)
try:
res = await power.locate(db, serial)
except RuntimeError as exc:
raise HTTPException(status_code=400, detail=str(exc))
if res["located"]:
await hub.publish("device.update", res["device"])
return res
@api.post("/devices/{serial}/power/{action}")
async def power_action(serial: str, action: str, request: Request):
db, hub = request.app.state.db, request.app.state.hub
await _device_or_404(db, serial)
if action not in ("on", "off", "cycle"):
raise HTTPException(status_code=404, detail="unknown power action")
_gate_idle()
# Free the port from any live serial monitor before toggling VBUS.
await request.app.state.serialmon.suspend(serial)
try:
result = await power.power_device(db, serial, action)
except AmbiguousPort as exc:
raise HTTPException(
status_code=409,
detail={"error": str(exc), "candidates": exc.candidates},
)
except (NoPort, ValueError) as exc:
raise HTTPException(status_code=400, detail=str(exc))
except RuntimeError as exc: # uhubctl errors (permissions, hub gone)
raise HTTPException(status_code=502, detail=str(exc))
finally:
if action != "off":
await request.app.state.serialmon.resume(serial)
return result
def _inject(size: int, port: str | None) -> dict:
return fixtures.push_fake_nodedb(
size, target="hardware", port=port, confirm=True, reboot_after=True
)
# --- cameras ---------------------------------------------------------------
def _mount_cameras(api: APIRouter) -> None:
@api.get("/cameras")
async def list_cameras(request: Request):
return await rc.list_all(request.app.state.db)
@api.get("/cameras/discover")
async def discover_cameras(request: Request):
# Indices already bound to a FleetSuite camera — don't re-open those.
in_use = {
str(c["device_index"])
for c in await rc.list_all(request.app.state.db)
if c.get("device_index") is not None
}
return await asyncio.to_thread(camera_stream.discover, in_use)
@api.post("/cameras")
async def add_camera(request: Request, body: dict = Body(...)):
db, hub = request.app.state.db, request.app.state.hub
cid = await rc.add(
db, name=body.get("name", "camera"), device_index=str(body.get("device_index", "0"))
)
cam = await rc.get(db, cid)
await hub.publish("camera.update", cam)
return cam
@api.delete("/cameras/{cid}", status_code=204)
async def remove_camera(cid: int, request: Request):
db, hub = request.app.state.db, request.app.state.hub
await rc.remove(db, cid)
await hub.publish("camera.update", {"id": cid, "deleted": True})
@api.post("/cameras/{cid}/assign")
async def assign_camera(cid: int, request: Request, body: dict = Body(...)):
db, hub = request.app.state.db, request.app.state.hub
cam = await rc.assign(db, cid, body.get("device_serial"))
if cam is None:
raise HTTPException(status_code=404, detail="unknown camera")
await hub.publish("camera.update", cam)
return cam
@api.post("/cameras/{cid}/rotation")
async def rotate_camera(cid: int, request: Request, body: dict = Body(...)):
db, hub = request.app.state.db, request.app.state.hub
cam = await rc.set_rotation(db, cid, int(body.get("rotation", 0)))
if cam is None:
raise HTTPException(status_code=404, detail="unknown camera")
await hub.publish("camera.update", cam)
return cam
@api.post("/cameras/{cid}/mirror")
async def mirror_camera(cid: int, request: Request, body: dict = Body(...)):
db, hub = request.app.state.db, request.app.state.hub
cam = await rc.set_mirror(db, cid, bool(body.get("mirror", False)))
if cam is None:
raise HTTPException(status_code=404, detail="unknown camera")
await hub.publish("camera.update", cam)
return cam
@api.get("/cameras/{cid}/status")
async def camera_status(cid: int, request: Request):
cam = await rc.get(request.app.state.db, cid)
if cam is None:
raise HTTPException(status_code=404, detail="unknown camera")
return await asyncio.to_thread(camera_stream.probe, str(cam.get("device_index")))
@api.get("/cameras/{cid}/stream.mjpg")
async def camera_stream_ep(cid: int, request: Request):
cam = await rc.get(request.app.state.db, cid)
if cam is None:
raise HTTPException(status_code=404, detail="unknown camera")
probe = await asyncio.to_thread(camera_stream.probe, str(cam.get("device_index")))
if not probe["ok"]:
raise HTTPException(status_code=503, detail=probe["error"])
return StreamingResponse(
camera_stream.mjpeg(str(cam.get("device_index"))),
media_type=f"multipart/x-mixed-replace; boundary={camera_stream.BOUNDARY}",
)
# --- firmware --------------------------------------------------------------
def _mount_firmware(api: APIRouter) -> None:
@api.get("/firmware")
async def get_firmware():
return await asyncio.to_thread(firmware.firmware_ref)
# --- builds ----------------------------------------------------------------
def _mount_builds(api: APIRouter) -> None:
@api.get("/builds")
async def list_builds(request: Request):
db = request.app.state.db
return {
"docker": await asyncio.to_thread(builder.docker_available),
"builds": await rb.list_all(db),
}
@api.post("/builds")
async def enqueue_builds(request: Request, body: dict = Body(default={})):
db = request.app.state.db
orch = request.app.state.orch
fw = await asyncio.to_thread(firmware.firmware_ref)
if not fw.get("available"):
raise HTTPException(status_code=400, detail="no firmware checkout")
sha, branch = fw["sha"], fw.get("branch")
envs = body.get("envs")
if not envs:
# Prebuild the envs every online, env-resolved device needs.
envs = sorted(
{control.env_for_device(d) for d in await rd.online_with_env(db)}
- {None}
)
if not envs:
return []
return await orch.enqueue(
list(envs), sha=sha, branch=branch, force=bool(body.get("force"))
)
# --- datadog ---------------------------------------------------------------
def _mount_datadog(api: APIRouter) -> None:
@api.get("/datadog")
async def get_datadog(request: Request):
return request.app.state.forwarder.status()
@api.put("/datadog")
async def put_datadog(request: Request, body: dict = Body(...)):
db = request.app.state.db
fwd = request.app.state.forwarder
cfg = await datadog.load_config(db)
for key in ("enabled", "site", "scrub", "collector", "host", "ship_debug"):
if key in body:
setattr(cfg, key, body[key])
# Only overwrite the key if a (non-empty) one was supplied.
if body.get("api_key"):
cfg.api_key = body["api_key"]
await datadog.save_config(db, cfg)
await fwd.reload()
status = fwd.status()
await request.app.state.hub.publish("datadog.update", status)
return status
@api.post("/datadog/test")
async def test_datadog(request: Request):
fwd = request.app.state.forwarder
await fwd.reload()
return await asyncio.to_thread(fwd.test_key)
# --- tests -----------------------------------------------------------------
def _mount_tests(api: APIRouter) -> None:
@api.get("/tests/status")
async def tests_status():
return test_runner.status()
@api.get("/tests/runs")
async def tests_runs(request: Request):
return await rr.list_runs(request.app.state.db)
@api.post("/tests/start")
async def tests_start(request: Request, body: dict = Body(default={})):
runner = request.app.state.runner
try:
return await runner.start(list(body.get("args", [])))
except RuntimeError as exc:
raise HTTPException(status_code=409, detail=str(exc))
@api.post("/tests/stop", status_code=204)
async def tests_stop(request: Request):
await request.app.state.runner.stop()
# --- native ----------------------------------------------------------------
def _mount_native(api: APIRouter) -> None:
@api.get("/native")
async def native_info(request: Request):
return await native.info(request.app.state.db)
@api.post("/native")
async def native_create(request: Request, body: dict = Body(...)):
db, hub = request.app.state.db, request.app.state.hub
name = (body.get("name") or "").strip()
if not name:
raise HTTPException(status_code=400, detail="missing name")
try:
dev = await native.create(db, name=name, tcp_port=int(body.get("tcp_port", 4403)))
except RuntimeError as exc:
raise HTTPException(status_code=400, detail=str(exc))
await hub.publish("device.update", dev)
return dev
@api.post("/native/{name}/{action}")
async def native_lifecycle(name: str, action: str, request: Request):
db, hub = request.app.state.db, request.app.state.hub
if action not in ("start", "stop", "restart"):
raise HTTPException(status_code=404, detail="unknown action")
try:
dev = await native.lifecycle(db, name, action)
except RuntimeError as exc:
raise HTTPException(status_code=400, detail=str(exc))
await hub.publish("device.update", dev)
return dev
@api.delete("/native/{name}", status_code=204)
async def native_delete(name: str, request: Request):
db, hub = request.app.state.db, request.app.state.hub
await native.remove(db, name)
await hub.publish("device.update", {"serial_number": f"native:{name}", "deleted": True})
# --- boards ----------------------------------------------------------------
def _mount_boards(api: APIRouter) -> None:
@api.get("/boards")
async def list_boards(query: str | None = None, architecture: str | None = None):
return await asyncio.to_thread(
boards.list_boards, architecture, False, query, None
)
# --- hubs (uhubctl) --------------------------------------------------------
def _mount_hubs(api: APIRouter) -> None:
@api.get("/hubs")
async def list_hubs():
if not power.available():
return {"available": False, "hubs": []}
try:
hubs = await asyncio.to_thread(power.list_hubs)
except RuntimeError as exc: # uhubctl present but failed (permissions)
return {"available": True, "hubs": [], "error": str(exc)}
return {"available": True, "hubs": hubs}
# --- websocket -------------------------------------------------------------
def _mount_ws(app: FastAPI) -> None:
@app.websocket("/ws")
async def ws(websocket: WebSocket):
await websocket.accept()
hub: Hub = websocket.app.state.hub
sm = websocket.app.state.serialmon
conn = Connection(send=websocket.send_json)
hub.add(conn)
# Track this peer's live serial monitors so we can release them on drop.
serials: set[str] = set()
try:
while True:
msg = await websocket.receive_json()
action = msg.get("action")
topic = msg.get("topic")
if action == "subscribe" and topic:
hub.subscribe(conn, topic)
if topic.startswith("serial.") and topic not in serials:
serials.add(topic)
await sm.acquire(topic[len("serial.") :])
elif action == "unsubscribe" and topic:
hub.unsubscribe(conn, topic)
if topic in serials:
serials.discard(topic)
await sm.release(topic[len("serial.") :])
except Exception: # noqa: BLE001 - normal on client disconnect
pass
finally:
hub.remove(conn)
for topic in serials:
await sm.release(topic[len("serial.") :])
@@ -1,3 +0,0 @@
"""SQLite registry for FleetSuite (aiosqlite). One ``Database`` owns the
connection; the ``repo_*`` modules are stateless helpers that take it as their
first argument."""
@@ -1,185 +0,0 @@
"""Thin async wrapper over a single aiosqlite connection.
``row_factory`` is set to ``aiosqlite.Row`` so every fetch behaves like a dict
(``row["col"]``); the ``repo_*`` modules copy rows into plain dicts before
handing them back so callers can add computed keys.
"""
from __future__ import annotations
import os
from pathlib import Path
import aiosqlite
# --- schema -----------------------------------------------------------------
# One file, created on connect. Plain `IF NOT EXISTS` — the registry is a cache
# of discovered hardware and recorded history, not a source of truth, so a
# wipe-and-rediscover is always safe.
SCHEMA = """
CREATE TABLE IF NOT EXISTS devices (
serial_number TEXT PRIMARY KEY,
node_num INTEGER,
friendly_name TEXT,
hw_model TEXT,
vid TEXT,
pid TEXT,
role TEXT,
current_port TEXT,
firmware_version TEXT,
region TEXT,
env TEXT,
env_locked INTEGER NOT NULL DEFAULT 0,
flashed_fw_branch TEXT,
flashed_fw_sha TEXT,
flashed_at REAL,
online INTEGER NOT NULL DEFAULT 0,
first_seen REAL NOT NULL DEFAULT 0,
last_seen REAL NOT NULL DEFAULT 0,
kind TEXT NOT NULL DEFAULT 'usb', -- 'usb' | 'native'
tcp_port INTEGER,
hub_location TEXT, -- uhubctl hub location (e.g. 1-1.3)
hub_port INTEGER -- uhubctl port number on that hub
);
CREATE TABLE IF NOT EXISTS cameras (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
type TEXT NOT NULL DEFAULT 'usb',
device_index TEXT,
backend TEXT,
rotation INTEGER NOT NULL DEFAULT 0,
mirror INTEGER NOT NULL DEFAULT 0,
enabled INTEGER NOT NULL DEFAULT 1,
created_at REAL NOT NULL DEFAULT 0,
device_serial TEXT,
assigned_at REAL
);
CREATE TABLE IF NOT EXISTS flash_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
device_serial TEXT NOT NULL,
env TEXT,
fw_sha TEXT,
from_artifact INTEGER NOT NULL DEFAULT 0,
duration_s REAL,
ok INTEGER NOT NULL DEFAULT 1,
ts REAL NOT NULL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS runs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
started_at REAL NOT NULL DEFAULT 0,
finished_at REAL,
exit_code INTEGER,
args TEXT,
seed TEXT,
fw_branch TEXT,
fw_sha TEXT,
fw_dirty INTEGER NOT NULL DEFAULT 0,
passed INTEGER NOT NULL DEFAULT 0,
failed INTEGER NOT NULL DEFAULT 0,
skipped INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS results (
id INTEGER PRIMARY KEY AUTOINCREMENT,
run_id INTEGER NOT NULL,
nodeid TEXT NOT NULL,
tier TEXT,
outcome TEXT,
duration_s REAL,
device_serial TEXT,
longrepr TEXT,
ts REAL NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_results_device ON results(device_serial);
CREATE INDEX IF NOT EXISTS idx_results_run ON results(run_id);
CREATE TABLE IF NOT EXISTS builds (
id INTEGER PRIMARY KEY AUTOINCREMENT,
env TEXT NOT NULL,
fw_sha TEXT NOT NULL,
fw_branch TEXT,
status TEXT NOT NULL DEFAULT 'queued',
duration_s REAL,
artifact_dir TEXT,
error TEXT,
created_at REAL NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_builds_key ON builds(env, fw_sha);
CREATE TABLE IF NOT EXISTS settings (
key TEXT PRIMARY KEY,
value TEXT
);
"""
def default_db_path() -> Path:
"""Where the registry lives for a normal (non-test) run."""
env = os.environ.get("MESHTASTIC_MCP_WEB_DB")
if env:
return Path(env)
return Path.home() / ".meshtastic_mcp" / "fleetsuite.db"
class Database:
"""Owns one aiosqlite connection. ``await connect()`` before use, and
``await close()`` when done. Helpers commit after every write — the access
pattern is low-volume bench traffic, not a hot loop."""
def __init__(self, path: Path | str) -> None:
self.path = Path(path)
self._conn: aiosqlite.Connection | None = None
async def connect(self) -> "Database":
self.path.parent.mkdir(parents=True, exist_ok=True)
self._conn = await aiosqlite.connect(str(self.path))
self._conn.row_factory = aiosqlite.Row
await self._conn.execute("PRAGMA journal_mode=WAL;")
await self._conn.executescript(SCHEMA)
await self._migrate()
await self._conn.commit()
return self
async def _migrate(self) -> None:
"""Additive column migrations for schema added after a db was first
created. ``ALTER TABLE ADD COLUMN`` is idempotent here — a duplicate on
a fresh db (already created with the column) is caught and ignored."""
additions = (
("cameras", "mirror", "INTEGER NOT NULL DEFAULT 0"),
("devices", "hub_location", "TEXT"),
("devices", "hub_port", "INTEGER"),
)
for table, col, decl in additions:
try:
await self._conn.execute(
f"ALTER TABLE {table} ADD COLUMN {col} {decl}"
)
except Exception: # noqa: BLE001 - column already present
pass
async def close(self) -> None:
if self._conn is not None:
await self._conn.close()
self._conn = None
@property
def conn(self) -> aiosqlite.Connection:
if self._conn is None:
raise RuntimeError("Database.connect() not called")
return self._conn
async def execute(self, sql: str, params: tuple = ()) -> aiosqlite.Cursor:
cur = await self.conn.execute(sql, params)
await self.conn.commit()
return cur
async def fetchone(self, sql: str, params: tuple = ()):
async with self.conn.execute(sql, params) as cur:
return await cur.fetchone()
async def fetchall(self, sql: str, params: tuple = ()):
async with self.conn.execute(sql, params) as cur:
return await cur.fetchall()
@@ -1,68 +0,0 @@
"""Firmware-build ledger, keyed by (env, fw_sha). The orchestrator records one
row per build attempt; ``get`` returns the latest for a key so a cache hit can
be distinguished from a fresh queue."""
from __future__ import annotations
import time
from .database import Database
_COLS = (
"id, env, fw_sha, fw_branch, status, duration_s, artifact_dir, error, "
"created_at"
)
def _to_dict(row) -> dict | None:
return dict(row) if row is not None else None
async def create(
db: Database, *, env: str, fw_sha: str, fw_branch: str | None, status: str
) -> int:
cur = await db.execute(
"INSERT INTO builds (env, fw_sha, fw_branch, status, created_at) "
"VALUES (?,?,?,?,?)",
(env, fw_sha, fw_branch, status, time.time()),
)
return cur.lastrowid
async def set_status(
db: Database,
build_id: int,
*,
status: str,
duration_s: float | None = None,
artifact_dir: str | None = None,
error: str | None = None,
) -> dict | None:
await db.execute(
"UPDATE builds SET status=?, duration_s=?, artifact_dir=?, error=? "
"WHERE id=?",
(status, duration_s, artifact_dir, error, build_id),
)
return await get_by_id(db, build_id)
async def get_by_id(db: Database, build_id: int) -> dict | None:
row = await db.fetchone(f"SELECT {_COLS} FROM builds WHERE id=?", (build_id,))
return _to_dict(row)
async def get(db: Database, env: str, fw_sha: str) -> dict | None:
"""Latest build row for a key, or None if it was never attempted."""
row = await db.fetchone(
f"SELECT {_COLS} FROM builds WHERE env=? AND fw_sha=? "
"ORDER BY id DESC LIMIT 1",
(env, fw_sha),
)
return _to_dict(row)
async def list_all(db: Database, limit: int = 100) -> list[dict]:
rows = await db.fetchall(
f"SELECT {_COLS} FROM builds ORDER BY id DESC LIMIT ?", (limit,)
)
return [dict(r) for r in rows]
@@ -1,81 +0,0 @@
"""Camera registry. A camera is an independent entity (a USB capture device)
that can be *assigned* to a device serial; rotation is a property of the camera
mount, so it survives reassignment."""
from __future__ import annotations
import time
from .database import Database
_COLS = (
"id, name, type, device_index, backend, rotation, mirror, enabled, "
"created_at, device_serial, assigned_at"
)
def _to_dict(row) -> dict | None:
return dict(row) if row is not None else None
def _normalize_rotation(rotation: int) -> int:
"""Snap to the nearest quarter-turn and wrap into [0, 360)."""
return int(round((rotation % 360) / 90.0)) * 90 % 360
async def add(db: Database, *, name: str, device_index: str) -> int:
cur = await db.execute(
"INSERT INTO cameras (name, type, device_index, rotation, enabled, "
"created_at) VALUES (?, 'usb', ?, 0, 1, ?)",
(name, device_index, time.time()),
)
return cur.lastrowid
async def get(db: Database, cid: int) -> dict | None:
row = await db.fetchone(f"SELECT {_COLS} FROM cameras WHERE id=?", (cid,))
return _to_dict(row)
async def list_all(db: Database) -> list[dict]:
rows = await db.fetchall(f"SELECT {_COLS} FROM cameras ORDER BY id")
return [dict(r) for r in rows]
async def remove(db: Database, cid: int) -> None:
await db.execute("DELETE FROM cameras WHERE id=?", (cid,))
async def assign(db: Database, cid: int, device_serial: str | None) -> dict | None:
await db.execute(
"UPDATE cameras SET device_serial=?, assigned_at=? WHERE id=?",
(device_serial, time.time() if device_serial else None, cid),
)
return await get(db, cid)
async def for_device(db: Database, serial: str) -> dict | None:
row = await db.fetchone(
f"SELECT {_COLS} FROM cameras WHERE device_serial=? LIMIT 1", (serial,)
)
return _to_dict(row)
async def set_rotation(db: Database, cid: int, rotation: int) -> dict | None:
await db.execute(
"UPDATE cameras SET rotation=? WHERE id=?",
(_normalize_rotation(rotation), cid),
)
return await get(db, cid)
async def set_mirror(db: Database, cid: int, mirror: bool) -> dict | None:
"""Horizontal flip — a property of the camera mount, like rotation."""
await db.execute(
"UPDATE cameras SET mirror=? WHERE id=?", (int(bool(mirror)), cid)
)
return await get(db, cid)
async def set_backend(db: Database, cid: int, backend: str | None) -> None:
await db.execute("UPDATE cameras SET backend=? WHERE id=?", (backend, cid))
@@ -1,222 +0,0 @@
"""Device registry — the heart of the harness's identity model.
A device is keyed by its USB serial number so its row (and everything attached
to it — friendly name, camera, pinned env, flash/test history) *follows* it
across ports and reboots. Discovery never deletes rows; it only flips ``online``
so a unplugged device greys out instead of vanishing.
"""
from __future__ import annotations
import time
from ..services import identity
from .database import Database
_COLS = (
"serial_number, node_num, friendly_name, hw_model, vid, pid, role, "
"current_port, firmware_version, region, env, env_locked, "
"flashed_fw_branch, flashed_fw_sha, flashed_at, online, first_seen, "
"last_seen, kind, tcp_port, hub_location, hub_port"
)
def _to_dict(row) -> dict | None:
if row is None:
return None
d = dict(row)
serial = d.get("serial_number") or ""
d["has_stable_id"] = identity.has_stable_id(serial)
# A device is "stale" once it has gone offline — the card stays but greys.
d["stale"] = not bool(d.get("online"))
return d
async def get(db: Database, serial: str) -> dict | None:
row = await db.fetchone(
f"SELECT {_COLS} FROM devices WHERE serial_number=?", (serial,)
)
return _to_dict(row)
async def list_all(db: Database) -> list[dict]:
rows = await db.fetchall(f"SELECT {_COLS} FROM devices ORDER BY first_seen")
return [_to_dict(r) for r in rows]
async def upsert_from_discovery(
db: Database,
*,
serial_number: str,
current_port: str,
vid: str | None,
pid: str | None,
role: str | None,
) -> dict:
"""Insert a freshly-seen device or refresh an existing one.
Returns the device dict with two transient flags the discovery loop uses to
decide what to broadcast: ``_is_new`` and ``_port_changed``. Never clobbers
operator-owned fields (friendly_name, pinned env) on re-discovery.
"""
now = time.time()
existing = await get(db, serial_number)
if existing is None:
await db.execute(
"INSERT INTO devices "
"(serial_number, current_port, vid, pid, role, kind, online, "
" first_seen, last_seen, env_locked) "
"VALUES (?,?,?,?,?,'usb',1,?,?,0)",
(serial_number, current_port, vid, pid, role, now, now),
)
row = await get(db, serial_number)
row["_is_new"] = True
row["_port_changed"] = False
return row
port_changed = existing["current_port"] != current_port
await db.execute(
"UPDATE devices SET current_port=?, vid=?, pid=?, role=?, online=1, "
"last_seen=? WHERE serial_number=?",
(current_port, vid, pid, role, now, serial_number),
)
row = await get(db, serial_number)
row["_is_new"] = False
row["_port_changed"] = port_changed
return row
async def upsert_native(
db: Database, *, name: str, tcp_port: int, online: bool = True
) -> dict:
"""A Docker ``meshtasticd`` node — surfaced as a device with a synthetic
``native:<name>`` serial so the same UI/card machinery applies."""
now = time.time()
serial = f"native:{name}"
port = f"tcp://127.0.0.1:{tcp_port}"
existing = await get(db, serial)
if existing is None:
await db.execute(
"INSERT INTO devices "
"(serial_number, friendly_name, role, current_port, kind, "
" tcp_port, online, first_seen, last_seen, env_locked) "
"VALUES (?,?, 'native', ?, 'native', ?, ?, ?, ?, 0)",
(serial, name, port, tcp_port, int(online), now, now),
)
else:
await db.execute(
"UPDATE devices SET tcp_port=?, current_port=?, online=?, "
"last_seen=? WHERE serial_number=?",
(tcp_port, port, int(online), now, serial),
)
return await get(db, serial)
async def set_friendly_name(db: Database, serial: str, name: str) -> dict | None:
await db.execute(
"UPDATE devices SET friendly_name=? WHERE serial_number=?", (name, serial)
)
return await get(db, serial)
async def set_env(
db: Database, serial: str, env: str | None, *, locked: bool
) -> dict | None:
"""Pin (``locked=True``) or release (``locked=False``) the pio env. A pinned
env is protected from auto-enrichment; releasing it lets detection win."""
await db.execute(
"UPDATE devices SET env=?, env_locked=? WHERE serial_number=?",
(env, int(locked), serial),
)
return await get(db, serial)
async def update_enrichment(
db: Database,
serial: str,
*,
node_num: int | None = None,
env: str | None = None,
hw_model: str | None = None,
firmware_version: str | None = None,
region: str | None = None,
) -> dict | None:
"""Apply data read off a connected device. ``env`` is only written when the
operator has NOT pinned one (``env_locked=0``); the rest always apply."""
row = await get(db, serial)
if row is None:
return None
sets = ["node_num=COALESCE(?, node_num)"]
params: list = [node_num]
for col, val in (
("hw_model", hw_model),
("firmware_version", firmware_version),
("region", region),
):
sets.append(f"{col}=COALESCE(?, {col})")
params.append(val)
if env is not None and not row["env_locked"]:
sets.append("env=?")
params.append(env)
params.append(serial)
await db.execute(
f"UPDATE devices SET {', '.join(sets)} WHERE serial_number=?", tuple(params)
)
return await get(db, serial)
async def set_hub_port(
db: Database, serial: str, *, location: str | None, port: int | None
) -> dict | None:
"""Pin (or clear) which uhubctl hub location + port this device sits on, so
power actions target the right physical port. Survives port/USB changes —
it's the hub slot, not the tty."""
await db.execute(
"UPDATE devices SET hub_location=?, hub_port=? WHERE serial_number=?",
(location, port, serial),
)
return await get(db, serial)
async def record_flashed(
db: Database, serial: str, *, branch: str | None, sha: str | None
) -> None:
await db.execute(
"UPDATE devices SET flashed_fw_branch=?, flashed_fw_sha=?, flashed_at=? "
"WHERE serial_number=?",
(branch, sha, time.time(), serial),
)
async def mark_offline_except(db: Database, keep: set[str]) -> list[str]:
"""Flip every currently-online device not in ``keep`` to offline. Returns
the serials that *transitioned* (so the caller can broadcast just those)."""
rows = await db.fetchall("SELECT serial_number FROM devices WHERE online=1")
newly = [r["serial_number"] for r in rows if r["serial_number"] not in keep]
for serial in newly:
await db.execute(
"UPDATE devices SET online=0 WHERE serial_number=?", (serial,)
)
return newly
async def online_by_role(db: Database, role: str) -> dict | None:
row = await db.fetchone(
f"SELECT {_COLS} FROM devices WHERE online=1 AND role=? "
"ORDER BY last_seen DESC LIMIT 1",
(role,),
)
return _to_dict(row)
async def online_with_env(db: Database) -> list[dict]:
"""Online, real (USB) devices that have a resolved env — the candidates the
test runner bakes per-board variant overrides from. Native nodes (no flash
target) and un-enriched devices (no env yet) are excluded."""
rows = await db.fetchall(
f"SELECT {_COLS} FROM devices "
"WHERE online=1 AND kind='usb' AND env IS NOT NULL"
)
return [_to_dict(r) for r in rows]
@@ -1,58 +0,0 @@
"""Flash-timing log. Each flash records whether it came from a prebuilt
artifact or a from-scratch host rebuild, so the UI can show how much the
artifact cache saves on a given device."""
from __future__ import annotations
import time
from .database import Database
_COLS = "id, device_serial, env, fw_sha, from_artifact, duration_s, ok, ts"
async def record(
db: Database,
*,
device_serial: str,
env: str | None,
fw_sha: str | None,
from_artifact: bool,
duration_s: float,
ok: bool,
) -> None:
await db.execute(
"INSERT INTO flash_events "
"(device_serial, env, fw_sha, from_artifact, duration_s, ok, ts) "
"VALUES (?,?,?,?,?,?,?)",
(
device_serial,
env,
fw_sha,
int(from_artifact),
duration_s,
int(ok),
time.time(),
),
)
async def _latest(db: Database, serial: str, from_artifact: bool) -> dict | None:
row = await db.fetchone(
f"SELECT {_COLS} FROM flash_events "
"WHERE device_serial=? AND from_artifact=? AND ok=1 "
"ORDER BY ts DESC LIMIT 1",
(serial, int(from_artifact)),
)
return dict(row) if row is not None else None
async def comparison(db: Database, serial: str) -> dict:
"""Latest successful artifact-flash vs latest successful host-rebuild flash,
plus the speedup ratio when both exist."""
artifact = await _latest(db, serial, True)
rebuild = await _latest(db, serial, False)
speedup = None
if artifact and rebuild and artifact["duration_s"]:
speedup = round(rebuild["duration_s"] / artifact["duration_s"], 1)
return {"artifact": artifact, "rebuild": rebuild, "speedup": speedup}
@@ -1,98 +0,0 @@
"""Test-run history. A run is a single pytest invocation; results are the
per-test leaves, optionally attributed to the device they exercised."""
from __future__ import annotations
import json
import time
from .database import Database
_RUN_COLS = (
"id, started_at, finished_at, exit_code, args, seed, fw_branch, fw_sha, "
"fw_dirty, passed, failed, skipped"
)
def _run_to_dict(row) -> dict | None:
if row is None:
return None
d = dict(row)
try:
d["args"] = json.loads(d["args"]) if d.get("args") else []
except (ValueError, TypeError):
d["args"] = []
return d
async def create_run(
db: Database,
*,
args: list[str],
seed: str | None,
fw_branch: str | None,
fw_sha: str | None,
fw_dirty: bool,
) -> int:
cur = await db.execute(
"INSERT INTO runs (started_at, args, seed, fw_branch, fw_sha, fw_dirty) "
"VALUES (?,?,?,?,?,?)",
(time.time(), json.dumps(args or []), seed, fw_branch, fw_sha, int(fw_dirty)),
)
return cur.lastrowid
async def get_run(db: Database, run_id: int) -> dict | None:
row = await db.fetchone(f"SELECT {_RUN_COLS} FROM runs WHERE id=?", (run_id,))
return _run_to_dict(row)
async def list_runs(db: Database, limit: int = 50) -> list[dict]:
rows = await db.fetchall(
f"SELECT {_RUN_COLS} FROM runs ORDER BY id DESC LIMIT ?", (limit,)
)
return [_run_to_dict(r) for r in rows]
async def finish_run(db: Database, run_id: int, *, exit_code: int | None) -> None:
await db.execute(
"UPDATE runs SET finished_at=?, exit_code=? WHERE id=?",
(time.time(), exit_code, run_id),
)
async def add_result(
db: Database,
run_id: int,
*,
nodeid: str,
tier: str | None,
outcome: str,
duration_s: float | None,
device_serial: str | None,
longrepr: str | None,
) -> None:
await db.execute(
"INSERT INTO results "
"(run_id, nodeid, tier, outcome, duration_s, device_serial, longrepr, ts) "
"VALUES (?,?,?,?,?,?,?,?)",
(run_id, nodeid, tier, outcome, duration_s, device_serial, longrepr, time.time()),
)
col = {"passed": "passed", "failed": "failed", "skipped": "skipped"}.get(outcome)
if col:
await db.execute(
f"UPDATE runs SET {col}={col}+1 WHERE id=?", (run_id,)
)
async def results_for_device(db: Database, serial: str) -> list[dict]:
"""Every recorded result for a device, newest first, joined to its run so
each row carries the firmware sha it ran against."""
rows = await db.fetchall(
"SELECT r.id, r.run_id, r.nodeid, r.tier, r.outcome, r.duration_s, "
"r.device_serial, r.longrepr, r.ts, runs.fw_sha, runs.fw_branch "
"FROM results r JOIN runs ON r.run_id = runs.id "
"WHERE r.device_serial=? ORDER BY r.id DESC",
(serial,),
)
return [dict(r) for r in rows]
@@ -1,26 +0,0 @@
"""Tiny key/value store for JSON-serialisable config blobs (e.g. the Datadog
forwarder settings)."""
from __future__ import annotations
import json
from .database import Database
async def set_json(db: Database, key: str, obj) -> None:
await db.execute(
"INSERT INTO settings (key, value) VALUES (?, ?) "
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
(key, json.dumps(obj)),
)
async def get_json(db: Database, key: str) -> dict | None:
row = await db.fetchone("SELECT value FROM settings WHERE key=?", (key,))
if row is None or row["value"] is None:
return None
try:
return json.loads(row["value"])
except (ValueError, TypeError):
return None
@@ -1,4 +0,0 @@
"""FleetSuite service layer — identity reconciliation, control gating, the
build orchestrator, the pytest runner, and the Datadog forwarder. These hold
the harness's domain logic; the REST/WS layer in ``app.py`` is a thin shell
over them."""
@@ -1,180 +0,0 @@
"""Firmware build orchestrator.
Builds are keyed by ``(env, fw_sha)`` and cached on disk under
``$MESHTASTIC_MCP_ARTIFACT_DIR/<sha>/<env>``. ``enqueue`` returns immediately
with a row per env (``status: queued`` for a fresh build, ``cached: True`` when
an artifact already exists); the actual compile runs in a background task so the
HTTP request never blocks. The compile itself is injectable (``build_fn``) so
the queue/cache/dedup logic is testable without Docker or a real toolchain.
"""
from __future__ import annotations
import asyncio
import logging
import os
import shutil
from pathlib import Path
from typing import Callable
from ..db import repo_builds as rb
log = logging.getLogger("meshtastic_mcp.web.builder")
# Signature of an injectable build function: compile ``env`` into ``out``,
# streaming progress through ``log_cb``; return True on success.
BuildFn = Callable[[str, Path, Callable[[str], None]], bool]
def _artifact_root() -> Path:
return Path(
os.environ.get(
"MESHTASTIC_MCP_ARTIFACT_DIR",
str(Path.home() / ".meshtastic_mcp" / "artifacts"),
)
)
def artifact_dir(sha: str, env: str) -> Path:
return _artifact_root() / sha / env
def cached_artifact_dir(sha: str, env: str) -> Path | None:
"""The cached artifact dir for a key if a successful build left one, else
None. 'Successful' is proven by the presence of a flashable image."""
d = artifact_dir(sha, env)
if d.is_dir() and (
any(d.glob("*.factory.bin")) or any(d.glob("*.bin")) or any(d.glob("*.uf2"))
):
return d
return None
def docker_available() -> bool:
if not shutil.which("docker"):
return False
try:
import subprocess
return (
subprocess.run(
["docker", "info"],
capture_output=True,
timeout=5,
).returncode
== 0
)
except Exception: # noqa: BLE001
return False
def default_build_fn(env: str, out: Path, log_cb: Callable[[str], None]) -> bool:
"""Host pio build, copying the resulting images into ``out``. Best-effort —
used when no build_fn is injected."""
try:
from meshtastic_mcp import config as mcfg, pio
root = mcfg.firmware_root()
log_cb(f"pio run -e {env}")
result = pio.run(["run", "-e", env], cwd=root, timeout=900, check=False)
log_cb(pio.tail_lines(result.stdout + result.stderr, 40))
if result.returncode != 0:
return False
out.mkdir(parents=True, exist_ok=True)
build_dir = root / ".pio" / "build" / env
copied = 0
for pattern in ("*.factory.bin", "*.bin", "*.uf2", "*.hex"):
for f in build_dir.glob(pattern):
shutil.copy2(f, out / f.name)
copied += 1
log_cb(f"copied {copied} artifact(s)")
return copied > 0
except Exception as exc: # noqa: BLE001
log_cb(f"build error: {exc}")
return False
class BuildOrchestrator:
def __init__(self, db, hub, build_fn: BuildFn | None = None) -> None:
self.db = db
self.hub = hub
self.build_fn = build_fn or default_build_fn
self._inflight: dict[str, asyncio.Task] = {}
async def enqueue(
self, envs: list[str], *, sha: str, branch: str | None, force: bool = False
) -> list[dict]:
"""Queue a build per env (skipping ones already cached or in flight).
``force`` rebuilds even when an artifact is cached. Returns a status row
per requested env."""
results: list[dict] = []
for env in envs:
key = f"{env}@{sha}"
cached = None if force else cached_artifact_dir(sha, env)
if cached is not None:
row = await rb.get(self.db, env, sha)
if row is None or row["status"] not in ("success", "cached"):
bid = await rb.create(
self.db, env=env, fw_sha=sha, fw_branch=branch, status="cached"
)
row = await rb.set_status(
self.db,
bid,
status="cached",
duration_s=0.0,
artifact_dir=str(cached),
)
results.append({**row, "cached": True})
await self.hub.publish("build.update", {**row, "cached": True})
continue
if key in self._inflight and not self._inflight[key].done():
row = await rb.get(self.db, env, sha)
if row:
results.append(row)
continue
bid = await rb.create(
self.db, env=env, fw_sha=sha, fw_branch=branch, status="queued"
)
row = await rb.get_by_id(self.db, bid)
results.append(row)
await self.hub.publish("build.update", row)
self._inflight[key] = asyncio.create_task(
self._run_build(bid, env, sha, key)
)
return results
async def _run_build(self, build_id: int, env: str, sha: str, key: str) -> None:
loop = asyncio.get_running_loop()
row = await rb.set_status(self.db, build_id, status="building")
await self.hub.publish("build.update", row)
out = artifact_dir(sha, env)
logs: list[str] = []
def log_cb(line: str) -> None:
logs.append(line)
self.hub.publish_threadsafe(
"build.update", {"id": build_id, "env": env, "log": line}
)
start = loop.time()
try:
ok = await asyncio.to_thread(self.build_fn, env, out, log_cb)
except Exception as exc: # noqa: BLE001
logs.append(f"exception: {exc}")
ok = False
duration = round(loop.time() - start, 2)
row = await rb.set_status(
self.db,
build_id,
status="success" if ok else "failed",
duration_s=duration,
artifact_dir=str(out) if ok else None,
error=None if ok else "\n".join(logs[-20:]) or "build failed",
)
await self.hub.publish("build.update", row)
self._inflight.pop(key, None)
@@ -1,170 +0,0 @@
"""MJPEG camera streaming.
Opens a capture device by OpenCV index and yields a ``multipart/x-mixed-replace``
JPEG stream for an ``<img>`` tag. Rotation is applied client-side (CSS), so the
stream is raw. OpenCV (the ``[ui]`` extra) is optional — without it, ``probe``
reports the error and the stream endpoint 503s instead of crashing.
"""
from __future__ import annotations
import asyncio
import json
import logging
import subprocess
import sys
from pathlib import Path
from typing import AsyncIterator
log = logging.getLogger("meshtastic_mcp.web.camera_stream")
BOUNDARY = "frame"
_FPS = 10.0
_MAX_INDEX = 10 # don't probe past this
def _import_cv2():
try:
import cv2 # type: ignore
try: # quiet the noisy "can't open camera" warnings during probing
cv2.utils.logging.setLogLevel(cv2.utils.logging.LOG_LEVEL_SILENT)
except Exception: # noqa: BLE001
pass
return cv2
except Exception: # noqa: BLE001
return None
def _enumerate_names() -> list[tuple[int, str]]:
"""Best-effort camera names from the OS, WITHOUT activating any device.
Returns ``[(index, name), ...]``. The index is the OpenCV capture index
(enumeration order on macOS, the videoN number on Linux)."""
if sys.platform == "darwin":
try:
out = subprocess.run(
["system_profiler", "SPCameraDataType", "-json"],
capture_output=True,
text=True,
timeout=10,
)
items = json.loads(out.stdout).get("SPCameraDataType", [])
return [(i, it.get("_name", f"camera {i}")) for i, it in enumerate(items)]
except Exception: # noqa: BLE001
return []
if sys.platform.startswith("linux"):
out: list[tuple[int, str]] = []
for node in sorted(Path("/sys/class/video4linux").glob("video*")):
try:
idx = int(node.name[len("video") :])
name = (node / "name").read_text().strip()
out.append((idx, name or f"video{idx}"))
except Exception: # noqa: BLE001
continue
return out
return []
def _probe_resolution(cv2, index: int) -> dict | None:
"""Open a capture index briefly to confirm it works + read its resolution.
Activates the camera momentarily; returns None if it won't open/read."""
try:
cap = cv2.VideoCapture(index)
except Exception: # noqa: BLE001
return None
try:
if not cap.isOpened():
return None
ok, _ = cap.read()
if not ok:
return None
return {
"width": int(cap.get(cv2.CAP_PROP_FRAME_WIDTH) or 0),
"height": int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0),
}
finally:
cap.release()
def discover(skip: set[str] | None = None, probe_resolution: bool = True) -> dict:
"""Enumerate attached cameras.
Name enumeration needs no OpenCV (works even without the ``[ui]`` extra), so
discovery is useful before streaming is installed. When cv2 IS present, each
not-in-use index is briefly opened to confirm it works and read its
resolution. ``skip`` is the set of indices already bound to a FleetSuite
camera — they're marked ``in_use`` and not re-opened (their stream owns them).
"""
skip = skip or set()
cv2 = _import_cv2()
named = _enumerate_names()
# If the OS gave us no names but cv2 is here, fall back to index probing.
if not named and cv2 is not None:
named = [(i, f"camera {i}") for i in range(_MAX_INDEX)]
cameras: list[dict] = []
for index, name in named:
if index >= _MAX_INDEX:
continue
entry: dict = {"index": index, "name": name, "in_use": str(index) in skip}
if cv2 is not None and not entry["in_use"] and probe_resolution:
res = _probe_resolution(cv2, index)
if res is None:
# Named by the OS but cv2 couldn't open it — surface, don't drop.
entry["unavailable"] = True
else:
entry.update(res)
cameras.append(entry)
return {
"available": True,
"cv2": cv2 is not None,
"cameras": cameras,
}
def probe(device_index: str) -> dict:
"""Can we open this device? Returns ``{ok, error}``."""
cv2 = _import_cv2()
if cv2 is None:
return {"ok": False, "error": "opencv not installed (pip install -e '.[ui]')"}
try:
cap = cv2.VideoCapture(int(device_index))
except (ValueError, TypeError):
return {"ok": False, "error": f"invalid device index: {device_index!r}"}
try:
if not cap.isOpened():
return {"ok": False, "error": "device did not open (in use or absent?)"}
ok, _ = cap.read()
return {"ok": bool(ok), "error": None if ok else "no frame from device"}
finally:
cap.release()
async def mjpeg(device_index: str) -> AsyncIterator[bytes]:
"""Async MJPEG frame generator. Reads happen in a worker thread so the event
loop is never blocked on the camera."""
cv2 = _import_cv2()
if cv2 is None:
return
cap = await asyncio.to_thread(cv2.VideoCapture, int(device_index))
try:
if not cap.isOpened():
return
while True:
ok, frame = await asyncio.to_thread(cap.read)
if not ok:
break
enc_ok, buf = await asyncio.to_thread(cv2.imencode, ".jpg", frame)
if enc_ok:
jpg = buf.tobytes()
yield (
b"--" + BOUNDARY.encode() + b"\r\n"
b"Content-Type: image/jpeg\r\n"
b"Content-Length: " + str(len(jpg)).encode() + b"\r\n\r\n"
+ jpg + b"\r\n"
)
await asyncio.sleep(1.0 / _FPS)
finally:
cap.release()
@@ -1,33 +0,0 @@
"""Device-control safety gate.
The central invariant: no ``connect()``-based action (flash, reboot, config,
send-text, factory-reset, nodedb inject) may run while a test run holds the
ports. Every control endpoint funnels through ``_ensure_idle`` /
``_ensure_port_free`` first, which raise ``ControlBusy`` (surfaced as HTTP 409)
when the runner is active.
"""
from __future__ import annotations
from . import identity, test_runner
class ControlBusy(RuntimeError):
"""Raised when a control action is attempted while a test run is active."""
def env_for_device(d: dict) -> str | None:
"""The pio env to flash/bake for a device: its resolved env if it has one,
otherwise the coarse role default."""
return d.get("env") or identity.env_for_role(d.get("role"))
def _ensure_idle() -> None:
if test_runner.is_running():
raise ControlBusy("a test run is in progress")
def _ensure_port_free(port: str | None) -> None:
# While a run is active it owns every port — nothing else may touch one.
if test_runner.is_running():
raise ControlBusy(f"port {port} is held by an active test run")
@@ -1,290 +0,0 @@
"""Datadog forwarder.
Pure mappers (``log_to_dd`` / ``telemetry_to_metrics``) turn recorder rows into
dashboard-compatible Datadog payloads; ``_read_live`` is a cursor-based tail of
the recorder's JSONL streams; ``DDConfig`` persists settings (with the API key
masked on the way out). The shipping loop lives in ``DDForwarder``.
"""
from __future__ import annotations
import json
import os
import re
from dataclasses import asdict, dataclass, fields
from pathlib import Path
import logging
from ..db import repo_settings as rs
from .scrub import Scrubber
log = logging.getLogger("meshtastic_mcp.web.datadog")
_ANSI = re.compile(r"\x1b\[[0-9;]*m")
_DDSOURCE = "meshtastic-firmware"
_STATUS = {
"ERROR": "error",
"CRIT": "error",
"CRITICAL": "error",
"WARN": "warning",
"WARNING": "warning",
"INFO": "info",
"DEBUG": "debug",
"TRACE": "debug",
"HEAP": "debug",
}
def _strip_ansi(s: str) -> str:
return _ANSI.sub("", s or "")
def _status_for(level: str | None) -> str:
if not level:
# Un-leveled output is a panic/backtrace — treat as error.
return "error"
return _STATUS.get(level.upper(), "info")
# --- log mapping ------------------------------------------------------------
def log_to_dd(
rec: dict,
*,
host: str,
base_tags: list[str],
port_tags: dict[str, list[str]],
scrubber: Scrubber,
ship_debug: bool,
) -> dict | None:
"""Map a recorder log row to a Datadog log intake payload.
Returns None for a DEBUG line when ``ship_debug`` is False. Un-leveled lines
(panics/backtraces) always ship.
"""
level = rec.get("level")
if level and level.upper() == "DEBUG" and not ship_debug:
return None
message = scrubber.scrub(_strip_ansi(rec.get("line", "")))
tags = list(base_tags)
port = rec.get("port")
if port:
tags.append(f"port:{port}")
tags.extend(port_tags.get(port, []))
if level:
tags.append(f"level:{level.lower()}")
tag = rec.get("tag")
if tag:
tags.append(f"thread:{tag.lower()}")
payload = {
"ddsource": _DDSOURCE,
"service": _DDSOURCE,
"hostname": host,
"message": message,
"ddtags": ",".join(tags),
"status": _status_for(level),
}
if level is not None:
payload["level"] = level
if rec.get("ts") is not None:
payload["timestamp"] = int(round(rec["ts"] * 1000))
if rec.get("heap_free") is not None:
payload["heap_free"] = rec["heap_free"]
return payload
# --- metric mapping ---------------------------------------------------------
def telemetry_to_metrics(
rec: dict,
*,
host: str,
base_tags: list[str],
port_tags: dict[str, list[str]],
) -> list[dict]:
"""Map a telemetry row to Datadog GAUGE series (one per numeric field).
Boolean and string fields are dropped."""
variant = rec.get("variant", "")
fields_ = rec.get("fields", {}) or {}
ts = int(rec.get("ts", 0) or 0)
tags = list(base_tags) + [f"variant:{variant}"]
port = rec.get("port")
if port:
tags.extend(port_tags.get(port, []))
out: list[dict] = []
for key, value in fields_.items():
# bool is a subclass of int — exclude it explicitly.
if isinstance(value, bool) or not isinstance(value, (int, float)):
continue
out.append(
{
"metric": f"mesh.{variant}.{key}",
"type": 3, # GAUGE
"points": [{"timestamp": ts, "value": float(value)}],
"resources": [{"type": "host", "name": host}],
"tags": list(tags),
}
)
return out
# --- cursor-based JSONL tail ------------------------------------------------
def _read_live(path: Path, cursor: dict, max_lines: int) -> tuple[list[dict], dict]:
"""Read newly-appended complete JSON lines from ``path``.
``cursor`` is ``{"ino": int, "pos": int}``. A partial trailing line (no
newline yet) is left for the next cycle. A cursor whose inode no longer
matches, or whose position is past EOF (rotation/truncation), resets to the
start. Returns ``(rows, next_cursor)``.
"""
path = Path(path)
try:
st = path.stat()
except OSError:
return [], dict(cursor)
ino = st.st_ino
pos = cursor.get("pos", 0) if cursor.get("ino") == ino else 0
if pos > st.st_size: # truncated or rotated under us
pos = 0
with open(path, "rb") as fh:
fh.seek(pos)
data = fh.read()
last_nl = data.rfind(b"\n")
if last_nl == -1:
# No complete line available — leave the cursor where it was.
return [], {"ino": ino, "pos": pos}
complete = data[: last_nl + 1]
consumed = pos + len(complete)
rows: list[dict] = []
for raw in complete.split(b"\n"):
if not raw.strip():
continue
try:
rows.append(json.loads(raw))
except ValueError:
continue
if len(rows) >= max_lines:
break
return rows, {"ino": ino, "pos": consumed}
# --- intake host derivation -------------------------------------------------
def _browser_intake_origin(site: str) -> str:
"""The RUM/browser-logs intake origin for a Datadog site.
``us5.datadoghq.com`` → ``https://browser-intake-us5-datadoghq.com``
``datadoghq.eu`` → ``https://browser-intake-datadoghq.eu``
"""
head, _, tld = site.rpartition(".")
head = head.replace(".", "-")
return f"https://browser-intake-{head}.{tld}" if head else f"https://browser-intake-{tld}"
def _logs_intake_url(site: str) -> str:
return f"https://http-intake.logs.{site}/api/v2/logs"
def _metrics_intake_url(site: str) -> str:
return f"https://api.{site}/api/v2/series"
# --- config -----------------------------------------------------------------
@dataclass
class DDConfig:
enabled: bool = False
api_key: str = ""
site: str = "datadoghq.com"
scrub: str = "coarse"
collector: str = ""
host: str = ""
ship_debug: bool = False
def masked(self) -> dict:
"""Config for the UI — the API key is never exposed, only a hint and a
client-token flag (Datadog client tokens start with ``pub``)."""
return {
"enabled": self.enabled,
"site": self.site,
"scrub": self.scrub,
"collector": self.collector,
"host": self.host,
"ship_debug": self.ship_debug,
"has_key": bool(self.api_key),
"key_hint": self.api_key[-4:] if self.api_key else "",
"is_client_token": self.api_key.startswith("pub") if self.api_key else False,
}
@classmethod
def from_dict(cls, d: dict | None) -> "DDConfig":
d = d or {}
allowed = {f.name for f in fields(cls)}
return cls(**{k: v for k, v in d.items() if k in allowed})
async def load_config(db) -> DDConfig:
return DDConfig.from_dict(await rs.get_json(db, "datadog"))
async def save_config(db, cfg: DDConfig) -> None:
await rs.set_json(db, "datadog", asdict(cfg))
# --- forwarder (runtime) ----------------------------------------------------
def _recorder_dir() -> Path:
return Path(os.environ.get("MESHTASTIC_MCP_RECORDER_DIR", ".mtlog"))
class DDForwarder:
"""Background loop that tails the recorder streams and ships to Datadog.
Started/reconfigured from the ``/api/datadog`` routes; a no-op until a key
and ``enabled`` are set."""
def __init__(self, db, hub) -> None:
self.db = db
self.hub = hub
self.cfg = DDConfig()
self.stats = {
"running": False,
"sent_logs": 0,
"sent_metrics": 0,
"cycles": 0,
"last_error": None,
"last_cycle_ts": None,
}
self._cursors: dict[str, dict] = {}
def status(self) -> dict:
return {"config": self.cfg.masked(), "stats": dict(self.stats)}
async def reload(self) -> None:
self.cfg = await load_config(self.db)
def test_key(self) -> dict:
"""Validate the configured API key against Datadog. Best-effort —
returns ``{ok, error}`` and never raises."""
if not self.cfg.api_key:
return {"ok": False, "error": "no API key configured"}
try:
import requests
resp = requests.get(
f"https://api.{self.cfg.site}/api/v1/validate",
headers={"DD-API-KEY": self.cfg.api_key},
timeout=10,
)
if resp.ok and resp.json().get("valid"):
return {"ok": True, "error": None}
return {"ok": False, "error": f"validation failed ({resp.status_code})"}
except Exception as exc: # noqa: BLE001 - surfaced to the UI
return {"ok": False, "error": str(exc)}
@@ -1,199 +0,0 @@
"""Background USB discovery loop.
Polls the serial bus, reconciles each likely-Meshtastic device into the
registry (keyed by stable serial, surrogate key otherwise), flips vanished
devices offline, and broadcasts the deltas on ``device.update``.
Auto-enrichment: when a device is newly seen (or hops ports, or hasn't been
read yet), the loop fires a one-shot ``device_info`` in the background to sniff
its firmware version, hw_model → exact pio env, region, and node num — so those
populate on their own at plug-in, FleetLog-style. It is gated hard for safety:
skipped entirely while a test run holds the ports, serialized so only one
device is connected at a time, the device's live serial monitor is suspended
for the moment of the connect, pinned envs are never clobbered, and failures
back off instead of re-hammering every poll. Disable with
``MESHTASTIC_MCP_AUTO_ENRICH=0``.
"""
from __future__ import annotations
import asyncio
import logging
import os
import time
from meshtastic_mcp import devices as devices_lib
from ..db import repo_devices as rd
from . import identity
log = logging.getLogger("meshtastic_mcp.web.discovery")
POLL_SECONDS = 4.0
ENRICH_BACKOFF_S = 60.0 # after a failed connect, wait this long before retrying
class DeviceDiscovery:
def __init__(self, db, hub, serialmon=None) -> None:
self.db = db
self.hub = hub
self.serialmon = serialmon
self.auto_enrich = os.environ.get("MESHTASTIC_MCP_AUTO_ENRICH", "1") != "0"
self._task: asyncio.Task | None = None
self._enrich_lock = asyncio.Lock() # one device on the wire at a time
self._enriched: dict[str, str] = {} # serial -> port last enriched at
self._failed: dict[str, float] = {} # serial -> monotonic time to retry after
self._enriching: set[str] = set() # in-flight, to dedupe schedules
def start(self) -> None:
if self._task is None:
self._task = asyncio.create_task(self._loop())
async def stop(self) -> None:
if self._task is not None:
self._task.cancel()
self._task = None
async def scan_once(self) -> None:
"""One discovery pass. Runs the (blocking) enumeration in a thread."""
try:
found = await asyncio.to_thread(devices_lib.list_devices)
except Exception as exc: # noqa: BLE001
log.debug("discovery enumeration failed: %s", exc)
return
seen: set[str] = set()
for dev in found:
if dev.get("blacklisted") or not dev.get("likely_meshtastic", False):
continue
key, _stable = identity.device_key(dev)
role = identity.role_for_vid(dev.get("vid"))
seen.add(key)
row = await rd.upsert_from_discovery(
self.db,
serial_number=key,
current_port=dev.get("port"),
vid=dev.get("vid"),
pid=dev.get("pid"),
role=role,
)
changed = bool(row.pop("_is_new", False)) | bool(
row.pop("_port_changed", False)
)
if changed:
await self.hub.publish("device.update", row)
self._maybe_enrich(row, changed)
# Keep native nodes alive across scans — they aren't on the USB bus.
natives = {
d["serial_number"]
for d in await rd.list_all(self.db)
if d.get("kind") == "native"
}
newly_offline = await rd.mark_offline_except(self.db, seen | natives)
for serial in newly_offline:
self._enriched.pop(serial, None) # re-verify if it comes back
row = await rd.get(self.db, serial)
if row:
await self.hub.publish("device.update", row)
# --- auto-enrichment --------------------------------------------------
def _maybe_enrich(self, row: dict, changed: bool) -> None:
"""Decide whether a discovered device needs a background enrichment and,
if so, schedule one. Cheap, synchronous gate; the actual connect happens
in :meth:`_enrich`."""
if not self.auto_enrich:
return
serial = row.get("serial_number")
if not serial or row.get("kind") == "native":
return
# Lazy import avoids a module-load cycle (test_runner ← control ← ...).
from . import test_runner
if test_runner.is_running():
return
if serial in self._enriching:
return
retry_at = self._failed.get(serial)
if retry_at is not None and time.monotonic() < retry_at:
return
# Enrich once per (serial, port). A completed-but-incomplete read sets a
# backoff (handled in _enrich), so we never reconnect every poll.
needs = changed or self._enriched.get(serial) != row.get("current_port")
if needs:
asyncio.create_task(self._enrich(serial))
async def _enrich(self, serial: str) -> None:
from meshtastic_mcp import info as mt_info
from . import test_runner
if serial in self._enriching:
return
self._enriching.add(serial)
try:
async with self._enrich_lock: # serialize: one port open at a time
if test_runner.is_running():
return
row = await rd.get(self.db, serial)
if (
row is None
or not row.get("online")
or row.get("kind") == "native"
):
return
port = row.get("current_port")
if not port:
return
# Free the port from any live serial monitor for the connect.
if self.serialmon is not None:
await self.serialmon.suspend(serial)
try:
info = await asyncio.to_thread(mt_info.device_info, port)
finally:
if self.serialmon is not None:
await self.serialmon.resume(serial)
hw_model = info.get("hw_model")
env = identity.env_for_hw_model(hw_model) if hw_model else None
updated = await rd.update_enrichment(
self.db,
serial,
node_num=info.get("my_node_num"),
env=env,
hw_model=str(hw_model) if hw_model else None,
firmware_version=info.get("firmware_version"),
region=info.get("region"),
)
if info.get("firmware_version"):
# Full read — terminal for this port.
self._enriched[serial] = port
self._failed.pop(serial, None)
log.info(
"enriched %s: fw=%s hw=%s env=%s",
serial,
info.get("firmware_version"),
hw_model,
env,
)
else:
# Connected but metadata wasn't ready yet — retry after backoff.
self._failed[serial] = time.monotonic() + ENRICH_BACKOFF_S
if updated:
await self.hub.publish("device.update", updated)
except Exception as exc: # noqa: BLE001 - a flaky device shouldn't kill the loop
self._failed[serial] = time.monotonic() + ENRICH_BACKOFF_S
log.debug("enrichment of %s failed (backing off): %s", serial, exc)
finally:
self._enriching.discard(serial)
async def _loop(self) -> None:
while True:
try:
await self.scan_once()
except asyncio.CancelledError:
raise
except Exception as exc: # noqa: BLE001
log.debug("discovery loop error: %s", exc)
await asyncio.sleep(POLL_SECONDS)
@@ -1,54 +0,0 @@
"""Current firmware git ref — branch, sha, dirty flag, and the tip commit's
subject/date. Read straight from git in the firmware root; degrades to
``{"available": False}`` outside a checkout."""
from __future__ import annotations
import subprocess
from functools import lru_cache
from pathlib import Path
@lru_cache(maxsize=1)
def _root() -> Path:
try:
from meshtastic_mcp import config as mcfg
return mcfg.firmware_root()
except Exception: # noqa: BLE001
return Path.cwd()
def _git(*args: str) -> str | None:
try:
out = subprocess.run(
["git", *args],
cwd=str(_root()),
capture_output=True,
text=True,
timeout=10,
)
if out.returncode != 0:
return None
return out.stdout.strip()
except Exception: # noqa: BLE001
return None
def firmware_ref() -> dict:
sha = _git("rev-parse", "HEAD")
if not sha:
return {"available": False}
branch = _git("rev-parse", "--abbrev-ref", "HEAD")
status = _git("status", "--porcelain")
subject = _git("log", "-1", "--pretty=%s")
committed_at = _git("log", "-1", "--pretty=%cI")
return {
"available": True,
"branch": branch,
"sha": sha,
"short_sha": sha[:7],
"dirty": bool(status),
"subject": subject,
"committed_at": committed_at,
}
@@ -1,87 +0,0 @@
"""Device identity reconciliation.
Two jobs: (1) map a USB VID to a coarse *role* and a role to a default pio
*env*, and resolve the precise env from a board's hw_model when we know it;
(2) derive a *stable key* for a device so a unit with a real serial number is
tracked across ports, while a serial-less unit gets a (port-derived) surrogate
that is explicitly NOT stable.
"""
from __future__ import annotations
import hashlib
from meshtastic_mcp import boards
# USB vendor IDs we recognise → coarse role. Case-insensitive; compared as the
# lowercased hex string (e.g. "0x239a").
_VID_ROLE = {
"0x239a": "nrf52", # Adafruit / RAK nRF52840
"0x303a": "esp32s3", # Espressif native USB
"0x10c4": "esp32s3", # Silicon Labs CP210x UART bridge
"0x1a86": "esp32s3", # WCH CH340/CH9102 UART bridge
}
# Coarse role → the default pio env to fall back on when we can't resolve the
# exact board variant from its hw_model.
_ROLE_ENV = {
"nrf52": "rak4631",
"esp32s3": "heltec-v3",
}
_NOSERIAL_PREFIX = "noserial:"
def role_for_vid(vid: str | None) -> str | None:
if not vid:
return None
return _VID_ROLE.get(vid.lower())
def env_for_role(role: str | None) -> str | None:
if not role:
return None
return _ROLE_ENV.get(role)
def env_for_hw_model(hw_model: str | None) -> str | None:
"""Resolve the exact pio env for a hardware model slug (e.g. ``HELTEC_V4``).
Prefers the *base* env (``heltec-v4``) over decorated variants
(``heltec-v4-tft``): when several envs declare the same hw_model slug, the
one whose name is the canonical slugification wins, else the shortest.
Returns None for an unknown slug.
"""
if not hw_model:
return None
target = hw_model.upper()
candidates = [
b["env"]
for b in boards.list_boards()
if (b.get("hw_model_slug") or "").upper() == target and b.get("env")
]
if not candidates:
return None
canonical = hw_model.lower().replace("_", "-")
if canonical in candidates:
return canonical
return min(candidates, key=len)
def device_key(d: dict) -> tuple[str, bool]:
"""Return ``(key, is_stable)`` for a discovered-device dict.
A real serial number is a stable key. Without one, we synthesise a
``noserial:<hash>`` surrogate from vid/pid/port so the device is still
addressable within a session — but it is NOT stable across replug.
"""
serial = d.get("serial_number")
if serial:
return str(serial), True
raw = f"{d.get('vid')}:{d.get('pid')}:{d.get('port')}"
digest = hashlib.sha1(raw.encode()).hexdigest()[:12]
return f"{_NOSERIAL_PREFIX}{digest}", False
def has_stable_id(key: str | None) -> bool:
return bool(key) and not str(key).startswith(_NOSERIAL_PREFIX)
@@ -1,86 +0,0 @@
"""Native (Docker ``meshtasticd``) node lifecycle.
Each native node is a container publishing the simulator's TCP API (4403) on a
host port, mirrored into the device registry as a ``native:<name>`` device so
the same card/UI applies. Best-effort: every Docker call is guarded and the
service reports ``docker: False`` rather than raising when Docker is absent.
"""
from __future__ import annotations
import asyncio
import logging
import shutil
from ..db import repo_devices as rd
from . import builder
log = logging.getLogger("meshtastic_mcp.web.native")
IMAGE = "meshtastic/meshtasticd:latest"
_PREFIX = "fleetsuite-native-"
def docker_available() -> bool:
return builder.docker_available()
async def _docker(*args: str, timeout: int = 30) -> tuple[int, str]:
if not shutil.which("docker"):
return 127, "docker not found"
proc = await asyncio.create_subprocess_exec(
"docker",
*args,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.STDOUT,
)
try:
out, _ = await asyncio.wait_for(proc.communicate(), timeout=timeout)
except asyncio.TimeoutError:
proc.kill()
return 124, "docker timed out"
return proc.returncode, out.decode(errors="replace")
def _container(name: str) -> str:
return f"{_PREFIX}{name}"
async def info(db) -> dict:
nodes = [
d for d in await rd.list_all(db) if d.get("kind") == "native"
]
return {"docker": docker_available(), "image": IMAGE, "nodes": nodes}
async def create(db, *, name: str, tcp_port: int) -> dict:
code, out = await _docker(
"run",
"-d",
"--name",
_container(name),
"-p",
f"{tcp_port}:4403",
IMAGE,
)
if code != 0:
raise RuntimeError(f"docker run failed: {out.strip()[:200]}")
return await rd.upsert_native(db, name=name, tcp_port=tcp_port, online=True)
async def lifecycle(db, name: str, action: str) -> dict:
verb = {"start": "start", "stop": "stop", "restart": "restart"}.get(action)
if verb is None:
raise ValueError(f"unknown action: {action}")
code, out = await _docker(verb, _container(name))
if code != 0:
raise RuntimeError(f"docker {verb} failed: {out.strip()[:200]}")
online = action != "stop"
row = await rd.get(db, f"native:{name}")
tcp_port = row.get("tcp_port") if row else 4403
return await rd.upsert_native(db, name=name, tcp_port=tcp_port, online=online)
async def remove(db, name: str) -> None:
await _docker("rm", "-f", _container(name))
await db.execute("DELETE FROM devices WHERE serial_number=?", (f"native:{name}",))
@@ -1,120 +0,0 @@
"""Per-node USB power control via uhubctl.
Each bench node sits on a PPPS-capable (per-port-power-switching) hub. uhubctl
can toggle VBUS on a given ``(hub_location, port)``, but its port listing only
exposes VID:PID — not the USB serial — so two same-VID nRF52s can't be told
apart from the hub side. We therefore track the mapping explicitly on the device
row (``hub_location`` / ``hub_port``):
* ``locate`` auto-binds a device when exactly one PPPS port matches its VID;
* an ambiguous match (two identical boards) is surfaced for the operator to
pick the right slot, which is then pinned and survives replug/reboot.
Power actions resolve through that mapping (falling back to a unique VID match)
and are gated by the run-safety lock like any other control action.
"""
from __future__ import annotations
import asyncio
import logging
import shutil
from pathlib import Path
from meshtastic_mcp import uhubctl
from ..db import repo_devices as rd
log = logging.getLogger("meshtastic_mcp.web.power")
_ACTIONS = {"on": uhubctl.power_on, "off": uhubctl.power_off, "cycle": uhubctl.cycle}
class AmbiguousPort(RuntimeError):
"""More than one PPPS port matches the device's VID — operator must pick."""
def __init__(self, candidates: list[dict]) -> None:
super().__init__("ambiguous hub port — assign one manually")
self.candidates = candidates
class NoPort(RuntimeError):
"""No controllable (PPPS) hub port hosts this device."""
def available() -> bool:
try:
from meshtastic_mcp import config as mcfg
binary = mcfg.uhubctl_bin()
if binary and Path(str(binary)).exists():
return True
except Exception: # noqa: BLE001
pass
return shutil.which("uhubctl") is not None
def _vid_int(device: dict) -> int | None:
vid = device.get("vid")
if not vid:
return None
try:
return int(str(vid), 16)
except ValueError:
return None
def list_hubs() -> list[dict]:
"""Parsed uhubctl hubs (with PPPS flag + per-port attachments)."""
return uhubctl.list_hubs()
def candidates_for(device: dict) -> list[dict]:
"""All PPPS ``(location, port)`` slots whose attached VID matches the
device, as ``[{"location", "port"}]``."""
vid = _vid_int(device)
if vid is None:
return []
return [
{"location": loc, "port": port}
for loc, port in uhubctl.find_port_for_vid(vid)
]
async def locate(db, serial: str) -> dict:
"""Try to auto-bind the device to its hub port. Returns the resolved
mapping, or the candidate list when ambiguous so the UI can prompt."""
device = await rd.get(db, serial)
if device is None:
raise LookupError(serial)
cands = await asyncio.to_thread(candidates_for, device)
if len(cands) == 1:
dev = await rd.set_hub_port(
db, serial, location=cands[0]["location"], port=cands[0]["port"]
)
return {"located": True, "device": dev, "candidates": cands}
return {"located": False, "device": device, "candidates": cands}
async def power_device(db, serial: str, action: str) -> dict:
"""Run a power action against the device's hub port. Resolves the pinned
mapping first, falling back to a unique VID match; raises ``AmbiguousPort``
/ ``NoPort`` when the slot can't be determined."""
if action not in _ACTIONS:
raise ValueError(f"unknown power action: {action}")
device = await rd.get(db, serial)
if device is None:
raise LookupError(serial)
loc = device.get("hub_location")
port = device.get("hub_port")
if loc is None or port is None:
cands = await asyncio.to_thread(candidates_for, device)
if not cands:
raise NoPort("no PPPS hub port hosts this device — assign one manually")
if len(cands) > 1:
raise AmbiguousPort(cands)
loc, port = cands[0]["location"], cands[0]["port"]
result = await asyncio.to_thread(_ACTIONS[action], loc, port)
return {"location": loc, "port": port, **result}
@@ -1,44 +0,0 @@
"""Coordinate scrubber for outbound telemetry/logs.
Three modes:
off — pass through unchanged
coarse — round decimal lat/lon to 1 dp, truncate 1e-7 integer coords to the
nearest million, drop NMEA sentence bodies
redact — replace any coordinate value with ``<scrubbed>``
Only coordinate-named keys are touched, so a decimal like ``latency=12.5`` is
left alone.
"""
from __future__ import annotations
import re
# Decimal coordinate: lat/lon/latitude/longitude = <signed decimal>. Longest
# names first so "latitude" isn't half-matched as "lat".
_DEC = re.compile(r"\b(latitude|longitude|lat|lon)=(-?\d+\.\d+)")
# Protobuf 1e-7 integer coordinate: same names with an _i suffix = <signed int>.
_INT = re.compile(r"\b(latitude_i|longitude_i|lat_i|lon_i)=(-?\d+)")
# NMEA sentence ("$GPGGA,...") — drop everything after the sentence type.
_NMEA = re.compile(r"(\$G[A-Z]{4}),\S*")
class Scrubber:
def __init__(self, mode: str = "coarse") -> None:
self.mode = (mode or "off").lower()
def scrub(self, text: str) -> str:
if self.mode == "off" or not text:
return text
if self.mode == "redact":
text = _DEC.sub(lambda m: f"{m.group(1)}=<scrubbed>", text)
text = _INT.sub(lambda m: f"{m.group(1)}=<scrubbed>", text)
text = _NMEA.sub(r"\1,<scrubbed>", text)
return text
# coarse
text = _DEC.sub(lambda m: f"{m.group(1)}={round(float(m.group(2)), 1)}", text)
text = _INT.sub(
lambda m: f"{m.group(1)}={int(float(m.group(2)) / 1e6) * 1_000_000}", text
)
text = _NMEA.sub(r"\1,<scrubbed>", text)
return text
@@ -1,142 +0,0 @@
"""Live serial monitors, one per device, multiplexed onto the ``serial.<serial>``
WebSocket topic.
A monitor is reference-counted by subscriber: the first client to open a
device's Serial tab spawns a pyserial reader thread that republishes each line;
the last to leave tears it down. Direct pyserial is used (not ``pio device
monitor``, whose miniterm backend requires a controlling TTY and crashes when
run headless under the server). Because the reader holds the USB port, any
control action ``suspend``s it for the duration and ``resume``s after, so the
port is never double-opened.
"""
from __future__ import annotations
import asyncio
import logging
import threading
import serial as pyserial
from meshtastic_mcp import connection
from ..db import repo_devices as rd
log = logging.getLogger("meshtastic_mcp.web.serial_monitor")
BAUD = 115200
class _Monitor:
def __init__(self) -> None:
self.refs = 0
self.suspended = False
self.stop: threading.Event | None = None
self.thread: threading.Thread | None = None
class SerialMonitor:
def __init__(self, db, hub) -> None:
self.db = db
self.hub = hub
self._mons: dict[str, _Monitor] = {}
def _topic(self, serial: str) -> str:
return f"serial.{serial}"
async def acquire(self, serial: str) -> None:
mon = self._mons.setdefault(serial, _Monitor())
mon.refs += 1
if mon.refs == 1 and not mon.suspended:
await self._open(serial, mon)
async def release(self, serial: str) -> None:
mon = self._mons.get(serial)
if mon is None:
return
mon.refs = max(0, mon.refs - 1)
if mon.refs == 0:
await self._close(mon)
self._mons.pop(serial, None)
async def suspend(self, serial: str) -> None:
"""Free the port for a control action (no-op if not monitored)."""
mon = self._mons.get(serial)
if mon is None:
return
mon.suspended = True
await self._close(mon)
async def resume(self, serial: str) -> None:
mon = self._mons.get(serial)
if mon is None:
return
mon.suspended = False
if mon.refs > 0 and mon.thread is None:
await self._open(serial, mon)
async def shutdown(self) -> None:
for mon in list(self._mons.values()):
await self._close(mon)
self._mons.clear()
async def _open(self, serial: str, mon: _Monitor) -> None:
row = await rd.get(self.db, serial)
if row is None or row.get("kind") == "native":
return # native nodes are TCP — nothing to monitor on the USB bus
port = row.get("current_port")
if not port or connection.is_tcp_port(port):
return
mon.stop = threading.Event()
mon.thread = threading.Thread(
target=self._read_loop, args=(serial, port, mon.stop), daemon=True
)
mon.thread.start()
async def _close(self, mon: _Monitor) -> None:
if mon.stop is not None:
mon.stop.set()
thread = mon.thread
mon.thread = None
mon.stop = None
if thread is not None:
await asyncio.to_thread(thread.join, 2.0)
def _read_loop(self, serial: str, port: str, stop: threading.Event) -> None:
"""Runs in a worker thread; publishes lines via the hub's thread-safe
path. Reads with a short timeout so ``stop`` is honoured promptly."""
topic = self._topic(serial)
try:
ser = pyserial.Serial(port, BAUD, timeout=0.5)
except Exception as exc: # noqa: BLE001
self.hub.publish_threadsafe(topic, {"line": f"— cannot open {port}: {exc}"})
return
self.hub.publish_threadsafe(topic, {"line": f"— monitor opened on {port}"})
buf = b""
try:
while not stop.is_set():
try:
data = ser.read(256)
except Exception as exc: # noqa: BLE001
self.hub.publish_threadsafe(topic, {"line": f"— read error: {exc}"})
break
if not data:
continue
buf += data
while b"\n" in buf:
raw, buf = buf.split(b"\n", 1)
text = raw.decode("utf-8", "replace").rstrip("\r")
if not text:
continue
# The meshtastic CDC carries protobuf API frames interleaved
# with text debug logs. Drop lines that are mostly undecodable
# bytes (a protobuf frame) — decoded text logs render with ANSI.
bad = text.count("")
if bad and bad > len(text) * 0.2:
continue
self.hub.publish_threadsafe(topic, {"line": text})
finally:
try:
ser.close()
except Exception: # noqa: BLE001
pass
@@ -1,267 +0,0 @@
"""The pytest runner.
``resolve_env_overrides`` and ``is_running`` are the pure pieces the safety
gate and the run-launcher depend on. ``TestRunner`` drives an actual pytest
subprocess: it bakes per-board env overrides, tails ``pytest-reportlog`` JSONL
for live per-test progress, and streams stdout/stderr + firmware logs over the
hub.
"""
from __future__ import annotations
import asyncio
import json
import logging
import os
import sys
import tempfile
import time
from pathlib import Path
log = logging.getLogger("meshtastic_mcp.web.test_runner")
# Tiers the UI knows about, in display order. A nodeid maps to a tier by its
# path under tests/ (directory name, or "bake"/"unit" for top-level files).
TIERS = (
"bake",
"unit",
"mesh",
"telemetry",
"monitor",
"fleet",
"admin",
"provisioning",
)
# Module-level run state. A single harness runs one suite at a time.
_state: dict = {"running": False, "run_id": None, "exit_code": None, "proc": None}
def is_running() -> bool:
return bool(_state.get("running"))
def status() -> dict:
return {
"running": _state["running"],
"run_id": _state["run_id"],
"exit_code": _state["exit_code"],
}
def resolve_env_overrides(rows: list[dict]) -> dict[str, str]:
"""From the online, env-resolved devices, bake one
``MESHTASTIC_MCP_ENV_<ROLE>=<env>`` override per role. Rows without a role
or env are skipped (so native/TCP nodes never become a flash target)."""
overrides: dict[str, str] = {}
for row in rows:
role = row.get("role")
env = row.get("env")
if not role or not env:
continue
overrides[f"MESHTASTIC_MCP_ENV_{role.upper()}"] = env
return overrides
def tier_for(nodeid: str) -> str:
"""Derive a tier from a pytest nodeid path."""
path = nodeid.split("::", 1)[0]
parts = path.split("/")
if "tests" in parts:
rest = parts[parts.index("tests") + 1 :]
if rest:
seg = rest[0]
if seg.endswith(".py"):
return "bake" if "bake" in seg else "unit"
return seg
return "unit"
def _split_nodeid(nodeid: str) -> tuple[str, str]:
path, _, name = nodeid.partition("::")
return path, name or nodeid
class TestRunner:
"""Owns the live pytest subprocess + its reportlog tail. One per app."""
def __init__(self, db, hub) -> None:
self.db = db
self.hub = hub
self._task: asyncio.Task | None = None
async def start(self, args: list[str]) -> dict:
from . import firmware # local import to avoid a cycle at module load
if is_running():
raise RuntimeError("a test run is already in progress")
fw = firmware.firmware_ref()
from ..db import repo_devices as rd
overrides = resolve_env_overrides(await rd.online_with_env(self.db))
from ..db import repo_runs as rr
run_id = await rr.create_run(
self.db,
args=args,
seed=str(int(time.time())),
fw_branch=fw.get("branch"),
fw_sha=fw.get("sha"),
fw_dirty=bool(fw.get("dirty")),
)
_state.update(running=True, run_id=run_id, exit_code=None)
await self.hub.publish("test.progress", {"type": "run_started", "run_id": run_id})
self._task = asyncio.create_task(self._drive(run_id, args, overrides))
return status()
async def stop(self) -> None:
proc = _state.get("proc")
if proc and proc.returncode is None:
try:
proc.terminate()
except ProcessLookupError:
pass
async def _drive(self, run_id: int, args: list[str], overrides: dict) -> None:
from .. import config as _cfg # noqa: F401
from ..db import repo_runs as rr
from meshtastic_mcp import config as mcfg
exit_code = None
report = Path(tempfile.gettempdir()) / f"fleetsuite-report-{run_id}.jsonl"
report.unlink(missing_ok=True)
try:
root = mcfg.firmware_root() / "mcp-server"
except Exception: # noqa: BLE001
root = Path.cwd()
env = dict(os.environ)
env.update(overrides)
cmd = [
sys.executable,
"-m",
"pytest",
"-p",
"no:cacheprovider",
f"--report-log={report}",
"-v",
*args,
]
try:
proc = await asyncio.create_subprocess_exec(
*cmd,
cwd=str(root),
env=env,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
_state["proc"] = proc
tail = asyncio.create_task(self._tail_report(run_id, report))
await asyncio.gather(
self._pump(proc.stdout, "stdout"),
self._pump(proc.stderr, "stderr"),
)
exit_code = await proc.wait()
await asyncio.sleep(0.2) # let the reportlog flush
tail.cancel()
except FileNotFoundError as exc:
await self.hub.publish(
"test.stdout", {"line": f"failed to launch pytest: {exc}", "source": "stderr"}
)
exit_code = 127
finally:
_state.update(running=False, exit_code=exit_code, proc=None)
await rr.finish_run(self.db, run_id, exit_code=exit_code)
await self.hub.publish(
"test.progress", {"type": "run_finished", "exit_code": exit_code}
)
report.unlink(missing_ok=True)
async def _pump(self, stream, source: str) -> None:
if stream is None:
return
while True:
raw = await stream.readline()
if not raw:
break
line = raw.decode(errors="replace").rstrip("\n")
await self.hub.publish("test.stdout", {"line": line, "source": source})
async def _tail_report(self, run_id: int, report: Path) -> None:
"""Follow the reportlog JSONL and translate entries into progress frames
+ persisted results."""
from ..db import repo_runs as rr
seen_register: set[str] = set()
pos = 0
while True:
if report.exists():
with open(report, "rb") as fh:
fh.seek(pos)
chunk = fh.read()
pos = fh.tell()
for raw in chunk.split(b"\n"):
if not raw.strip():
continue
try:
entry = json.loads(raw)
except ValueError:
continue
await self._handle_entry(run_id, entry, seen_register, rr)
await asyncio.sleep(0.3)
async def _handle_entry(self, run_id, entry, seen_register, rr) -> None:
rtype = entry.get("$report_type")
if rtype == "CollectReport":
for item in entry.get("result", []):
nodeid = item.get("nodeid")
if not nodeid or "::" not in nodeid or nodeid in seen_register:
continue
seen_register.add(nodeid)
path, name = _split_nodeid(nodeid)
await self.hub.publish(
"test.progress",
{
"type": "register",
"nodeid": nodeid,
"tier": tier_for(nodeid),
"file": path,
"testname": name,
},
)
elif rtype == "TestReport":
nodeid = entry.get("nodeid")
when = entry.get("when")
outcome = entry.get("outcome")
if when == "setup":
await self.hub.publish(
"test.progress", {"type": "running", "nodeid": nodeid}
)
# Final outcome: the call phase normally, or a non-passed setup
# (skip/error) that short-circuits the test.
final = when == "call" or (when == "setup" and outcome != "passed")
if final:
duration = entry.get("duration")
await self.hub.publish(
"test.progress",
{
"type": "outcome",
"nodeid": nodeid,
"outcome": outcome,
"duration": duration,
},
)
longrepr = entry.get("longrepr")
await rr.add_result(
self.db,
run_id,
nodeid=nodeid,
tier=tier_for(nodeid or ""),
outcome=outcome,
duration_s=duration,
device_serial=None,
longrepr=str(longrepr) if longrepr else None,
)

Some files were not shown because too many files have changed in this diff Show More