Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9dc76944c1 |
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"version": "0.0.1",
|
||||
"configurations": [
|
||||
{
|
||||
"name": "fleetsuite-ui",
|
||||
"runtimeExecutable": "npm",
|
||||
"runtimeArgs": [
|
||||
"--prefix",
|
||||
"mcp-server/web-ui",
|
||||
"run",
|
||||
"dev",
|
||||
"--",
|
||||
"--port",
|
||||
"5199",
|
||||
"--strictPort"
|
||||
],
|
||||
"port": 5199
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
{
|
||||
"hooks": {
|
||||
"PostToolUse": [
|
||||
{
|
||||
"matcher": "Write|Edit",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "f=$(tr -d '\\n' | grep -o '\"file_path\"[[:space:]]*:[[:space:]]*\"[^\"]*\"' | head -1 | sed 's/.*:[[:space:]]*\"//; s/\"$//'); [ -n \"$f\" ] && [ -f \"$f\" ] || exit 0; t=$(command -v trunk || echo \"$HOME/.cache/trunk/launcher/trunk\"); [ -x \"$t\" ] || { echo \"trunk-fmt hook: trunk not found; its launcher needs curl or wget to bootstrap the CLI (see 'Formatting & the trunk toolchain' in .github/copilot-instructions.md)\" >&2; exit 1; }; out=$(\"$t\" fmt --force \"$f\" 2>&1) || { echo \"trunk-fmt hook: trunk fmt failed on $f: $out\" >&2; exit 1; }",
|
||||
"timeout": 120,
|
||||
"statusMessage": "Formatting (trunk)..."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -18,7 +18,7 @@ ENV PIP_ROOT_USER_ACTION=ignore
|
||||
# trunk-ignore(hadolint/DL3008): apt packages are not pinned.
|
||||
# trunk-ignore(terrascan/AC_DOCKER_0002): apt packages are not pinned.
|
||||
RUN apt-get update && apt-get install --no-install-recommends -y \
|
||||
cmake git zip libgpiod-dev libjsoncpp-dev libbluetooth-dev libi2c-dev \
|
||||
cmake git zip libgpiod-dev libbluetooth-dev libi2c-dev \
|
||||
libunistring-dev libmicrohttpd-dev libgnutls28-dev libgcrypt20-dev \
|
||||
libusb-1.0-0-dev libssl-dev pkg-config libsqlite3-dev libsdl2-dev && \
|
||||
apt-get clean && rm -rf /var/lib/apt/lists/* && \
|
||||
|
||||
@@ -31,7 +31,7 @@ cmake --install "$WORK/ulfius/$SANITIZER" --prefix /usr
|
||||
cd "$SRC/firmware"
|
||||
|
||||
PLATFORMIO_EXTRA_SCRIPTS=$(echo -e "pre:.clusterfuzzlite/platformio-clusterfuzzlite-pre.py\npost:.clusterfuzzlite/platformio-clusterfuzzlite-post.py")
|
||||
STATIC_LIBS=$(pkg-config --libs --static libulfius openssl libgpiod yaml-cpp jsoncpp bluez --silence-errors)
|
||||
STATIC_LIBS=$(pkg-config --libs --static libulfius openssl libgpiod yaml-cpp bluez --silence-errors)
|
||||
export PLATFORMIO_EXTRA_SCRIPTS
|
||||
export STATIC_LIBS
|
||||
export PLATFORMIO_WORKSPACE_DIR="$WORK/pio/$SANITIZER"
|
||||
|
||||
@@ -16,7 +16,6 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
|
||||
libssl-dev \
|
||||
libulfius-dev \
|
||||
libyaml-cpp-dev \
|
||||
libjsoncpp-dev \
|
||||
pipx \
|
||||
pkg-config \
|
||||
python3 \
|
||||
|
||||
@@ -8,21 +8,17 @@
|
||||
"features": {
|
||||
"ghcr.io/devcontainers/features/python:1": {
|
||||
"installTools": true,
|
||||
"version": "3.13"
|
||||
"version": "3.14"
|
||||
}
|
||||
},
|
||||
"customizations": {
|
||||
"vscode": {
|
||||
"extensions": [
|
||||
"ms-vscode.cpptools",
|
||||
"Jason2866.esp-decoder",
|
||||
"pioarduino.pioarduino-ide",
|
||||
"platformio.platformio-ide",
|
||||
"Trunk.io"
|
||||
],
|
||||
"unwantedRecommendations": [
|
||||
"ms-azuretools.vscode-docker",
|
||||
"platformio.platformio-ide"
|
||||
],
|
||||
"unwantedRecommendations": ["ms-azuretools.vscode-docker"],
|
||||
"settings": {
|
||||
"extensions.ignoreRecommendations": true
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ runs:
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt-get -y update --fix-missing
|
||||
sudo apt-get install -y cppcheck libbluetooth-dev libgpiod-dev libyaml-cpp-dev libjsoncpp-dev lsb-release
|
||||
sudo apt-get install -y cppcheck libbluetooth-dev libgpiod-dev libyaml-cpp-dev lsb-release
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v6
|
||||
|
||||
@@ -11,4 +11,4 @@ runs:
|
||||
- name: Install libs needed for native build
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt-get install -y libbluetooth-dev libgpiod-dev libyaml-cpp-dev libjsoncpp-dev openssl libssl-dev libulfius-dev liborcania-dev libusb-1.0-0-dev libi2c-dev libuv1-dev
|
||||
sudo apt-get install -y libbluetooth-dev libgpiod-dev libyaml-cpp-dev openssl libssl-dev libulfius-dev liborcania-dev libusb-1.0-0-dev libi2c-dev libuv1-dev
|
||||
|
||||
@@ -135,99 +135,6 @@ On top of authorization, any remote admin message that **mutates** state (not a
|
||||
- **Channel 0 PSK change** → every peer must re-learn the channel hash; cached NodeInfo becomes temporarily unreachable until the next broadcast.
|
||||
- **`security.private_key` blanked via admin** → regenerates both halves (unless in Ham mode) and propagates the new public key via NodeInfo.
|
||||
|
||||
## NodeDB Layout (v25)
|
||||
|
||||
`DEVICESTATE_CUR_VER = 25`, `DEVICESTATE_MIN_VER = 24`. The on-device NodeDB was split in v25 into a slim header table plus four optional satellite stores. Older v24 saves auto-migrate at boot. Old training-data instincts (`node->user.long_name`, `node->position.latitude_i`, `node->is_favorite`, `node->device_metrics.battery_level`) are wrong now — the fields aren't there. Read this section before touching anything that walks `nodeDB->meshNodes`.
|
||||
|
||||
### Slim `NodeInfoLite`
|
||||
|
||||
`UserLite` is flattened onto `NodeInfoLite` (no nested sub-message); `position` and `device_metrics` are removed entirely (tags reserved). MAC address is dropped. Long names are capped at 25 chars (`max_size:25` in `deviceonly.options`); `hw_model` and `role` are `int_size:8`. Encoded size dropped from ~166 B → ~105 B per node.
|
||||
|
||||
Booleans are bit-packed into `NodeInfoLite.bitfield`. **Do not read or write the bits directly** — use the inline helpers in `src/mesh/NodeDB.h`:
|
||||
|
||||
```cpp
|
||||
nodeInfoLiteHasUser(n) // bit 5 — user fields populated
|
||||
nodeInfoLiteIsFavorite(n) // bit 3
|
||||
nodeInfoLiteIsIgnored(n) // bit 4
|
||||
nodeInfoLiteIsMuted(n) // bit 1
|
||||
nodeInfoLiteIsLicensed(n) // bit 6 — Ham mode peer
|
||||
nodeInfoLiteIsKeyManuallyVerified(n) // bit 0
|
||||
nodeInfoLiteHasIsUnmessagable(n) // bit 8 — "is_unmessagable was sent"
|
||||
nodeInfoLiteIsUnmessagable(n) // bit 7
|
||||
// via_mqtt is bit 2 (mask exposed; predicate uses the mask directly)
|
||||
|
||||
nodeInfoLiteSetBit(n, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true); // setter
|
||||
```
|
||||
|
||||
### Satellite stores
|
||||
|
||||
Four `std::unordered_map<NodeNum, …>` members on `NodeDB`, each gated by its own build flag:
|
||||
|
||||
| Map | Value type | Build flag |
|
||||
| ----------------- | ------------------------------- | ---------------------------------- |
|
||||
| `nodePositions` | `meshtastic_PositionLite` | `MESHTASTIC_EXCLUDE_POSITIONDB` |
|
||||
| `nodeTelemetry` | `meshtastic_DeviceMetrics` | `MESHTASTIC_EXCLUDE_TELEMETRYDB` |
|
||||
| `nodeEnvironment` | `meshtastic_EnvironmentMetrics` | `MESHTASTIC_EXCLUDE_ENVIRONMENTDB` |
|
||||
| `nodeStatus` | `meshtastic_StatusMessage` | `MESHTASTIC_EXCLUDE_STATUSDB` |
|
||||
|
||||
Defaults are ON (i.e., maps **excluded**) for STM32WL only — see `src/mesh/mesh-pb-constants.h`. On every other arch all four maps are present. When excluded, the map member is absent and the corresponding accessors return `false`.
|
||||
|
||||
All four maps are guarded by **`mutable concurrency::Lock satelliteMutex`** — concurrent access from receive threads, the phone API state machine, and the renderer is the rule, not the exception.
|
||||
|
||||
### Accessor convention
|
||||
|
||||
**Never hand out pointers into the maps.** Use the copy-out accessors on `NodeDB`:
|
||||
|
||||
```cpp
|
||||
bool copyNodePosition(NodeNum, meshtastic_PositionLite &out) const;
|
||||
bool copyNodeTelemetry(NodeNum, meshtastic_DeviceMetrics &out) const;
|
||||
bool copyNodeEnvironment(NodeNum, meshtastic_EnvironmentMetrics &out) const;
|
||||
bool copyNodeStatus(NodeNum, meshtastic_StatusMessage &out) const;
|
||||
```
|
||||
|
||||
Each takes the lock, copies the value if present, returns `false` if the entry is absent or the DB is excluded. Pass-by-out-param is deliberate — pointer-style accessors would invite UAF and lock-leak bugs across the renderer. The "has any X" convenience predicates (`hasValidPosition` etc.) are implemented in terms of these.
|
||||
|
||||
Writers go through `setNodeStatus`, `updatePosition`, `updateTelemetry` (which dispatches on `which_variant` for device vs environment metrics) — these own the lock and the eviction hooks.
|
||||
|
||||
### Eviction
|
||||
|
||||
Every code path that drops a node from the header table must also evict the satellites. The single chokepoint is `eraseNodeSatellites(NodeNum)`; it's already called from `getOrCreateMeshNode`'s oldest-boring eviction, `removeNodeByNum`, both branches of `resetNodes`, `cleanupMeshDB`, `addFromContact`'s ignored-branch, and `AdminModule`'s `set_ignored_node`. Add new eviction sites here, not by calling `.erase()` directly.
|
||||
|
||||
### Sync flow: thin NodeInfo + post-COMPLETE_ID replay (no opt-in)
|
||||
|
||||
There is no capability flag and no special "gradient" nonce. The **default** sync flow is:
|
||||
|
||||
1. Config / module-config / channel / metadata segments (same as before).
|
||||
2. `STATE_SEND_OWN_NODEINFO` — **our own** NodeInfo, still bundled with our position and device_metrics (because the replay snapshot excludes our own NodeNum). Emitted via `ConvertToNodeInfo(lite)`.
|
||||
3. `STATE_SEND_OTHER_NODEINFOS` — every other peer's NodeInfo, **always thin** (no `position`, no `device_metrics`). Emitted via `ConvertToNodeInfoThin(lite)`.
|
||||
4. `STATE_SEND_FILEMANIFEST` → `STATE_SEND_COMPLETE_ID` — the phone sees `config_complete_id` and treats sync as done.
|
||||
5. `STATE_SEND_PACKETS` — live mesh packets, with a trailing replay drain interleaved. The replay drain walks four cached satellite stores in order (positions → telemetry → environment → status) and emits each cached entry as an ordinary `MeshPacket` on the matching portnum (`POSITION_APP`, `TELEMETRY_APP` device + environment variants, `NODE_STATUS_APP`). These are indistinguishable on the wire from live mesh traffic, so clients need no special handling — any code that already updates UI on `POSITION_APP` etc. works.
|
||||
|
||||
`PhoneAPI::sendConfigComplete()` arms `replayPhase = REPLAY_PHASE_POSITIONS` for default/full sync and `SPECIAL_NONCE_ONLY_NODES`, while `SPECIAL_NONCE_ONLY_CONFIG` skips replay. The drain runs inside `STATE_SEND_PACKETS` via `popReplayPacket()`, lower priority than live traffic. When all four phases drain, `replayPhase` flips back to `REPLAY_PHASE_IDLE` and the snapshot vectors get `shrink_to_fit`ed.
|
||||
|
||||
STM32WL and any other build with all four `MESHTASTIC_EXCLUDE_*DB` flags set produces zero replay packets — `popReplayPacket` advances through each phase in microseconds without emitting anything.
|
||||
|
||||
Special nonces that still mean something:
|
||||
|
||||
- `SPECIAL_NONCE_ONLY_CONFIG` (69420) — skip node sync entirely, just config.
|
||||
- `SPECIAL_NONCE_ONLY_NODES` (69421) — skip config segments, jump straight to `STATE_SEND_OWN_NODEINFO`. Still gets the post-COMPLETE_ID replay drain.
|
||||
|
||||
There are no other reserved nonces; everything else is a fresh random `want_config_id` from the client.
|
||||
|
||||
### v24 → v25 migration
|
||||
|
||||
The legacy migration code lives in **`src/mesh/NodeDBLegacyMigration.cpp`**, not in `NodeDB.cpp`. It owns the `meshtastic_NodeDatabase_Legacy` callback and `NodeDB::migrateLegacyNodeDatabase()`. The legacy proto descriptor is `protobufs/meshtastic/deviceonly_legacy.proto` (only included by the migration TU). The boot path peeks the file's leading version tag, runs the migration if `version < 25`, then re-saves in v25 layout. The legacy descriptor is scheduled for removal once `DEVICESTATE_MIN_VER` is bumped.
|
||||
|
||||
### Read-site rules of thumb
|
||||
|
||||
- Never `node->position.X` / `node->device_metrics.X` — those fields no longer exist. Pull from the satellite map via `copyNodePosition` / `copyNodeTelemetry`.
|
||||
- Never `node->user.long_name` — `long_name`, `short_name`, `public_key`, `hw_model`, `role`, `macaddr` (gone), `is_licensed`, `is_unmessagable` are flat on `NodeInfoLite`.
|
||||
- Never `node->is_favorite` / `node->is_ignored` / `node->via_mqtt` / `node->is_key_manually_verified` — use the bitfield helpers.
|
||||
- Never assume `nodeDB->getMeshNode(num)->position.time` — call `copyNodePosition` and check the return.
|
||||
- Don't lock `satelliteMutex` yourself in renderer code; the copy-out accessors already do.
|
||||
|
||||
Unit tests for the conversion layer live in `test/test_type_conversions/test_main.cpp` (Unity) — bitfield round-trips, `long_name` truncation, thin-vs-full conversions. Add cases there when extending the schema.
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
@@ -283,15 +190,6 @@ firmware/
|
||||
|
||||
## Coding Conventions
|
||||
|
||||
### Formatting & the trunk toolchain
|
||||
|
||||
`trunk fmt` is the project formatter (`trunk_check` CI rejects unformatted code). For Claude Code users, `.claude/settings.json` ships a PostToolUse hook that runs `trunk fmt --force` on every file the agent writes or edits. The hook is pure sh/grep/sed — no python or jq required — but trunk itself must be able to run:
|
||||
|
||||
- Trunk's launcher (`~/.cache/trunk/launcher/trunk`, or `trunk` on PATH) downloads the CLI version pinned in `.trunk/trunk.yaml` on first use and again whenever that pin is bumped. **The launcher needs `curl` or `wget`**; without one it fails with "Cannot download… please install curl or wget", and the hook surfaces that as a warning on every write.
|
||||
- No curl/wget available (e.g. a minimal WSL image)? Bootstrap by hand with any Python (PlatformIO bundles one at `~/.platformio/penv/bin/python`): download `https://trunk.io/releases/<ver>/trunk-<ver>-linux-x86_64.tar.gz` and place the `trunk` binary at `~/.cache/trunk/cli/<ver>-linux-x86_64/trunk` (chmod +x), where `<ver>` is the `cli.version` from `.trunk/trunk.yaml`.
|
||||
- The hook fails loudly by design (visible warning, non-blocking). Silent no-op formatting hooks hide real breakage — don't re-add `2>/dev/null || true` around the whole thing.
|
||||
- More generally: don't assume a stock Linux userland in hooks or helper scripts — minimal WSL/container images may lack `python3`, `curl`, `wget`, and `jq`. Prefer plain sh + coreutils, or PlatformIO's bundled Python for anything heavier.
|
||||
|
||||
### General Style
|
||||
|
||||
- Follow existing code style - run `trunk fmt` before commits
|
||||
@@ -618,35 +516,20 @@ Most workflows can be triggered manually via `workflow_dispatch` for testing.
|
||||
|
||||
### Native unit tests (C++)
|
||||
|
||||
Unit tests in `test/` directory with 17 test suites:
|
||||
Unit tests in `test/` directory with 12 test suites:
|
||||
|
||||
- `test_admin_radio/` - LoRa region/config validation and AdminModule dispatch
|
||||
- `test_atak/` - ATAK integration
|
||||
- `test_crypto/` - Cryptography
|
||||
- `test_default/` - Default configuration
|
||||
- `test_http_content_handler/` - HTTP handling
|
||||
- `test_mac_from_string/` - MAC address parsing
|
||||
- `test_mqtt/` - MQTT integration
|
||||
- `test_radio/` - Radio interface
|
||||
- `test_mesh_module/` - Module framework
|
||||
- `test_meshpacket_serializer/` - Packet serialization
|
||||
- `test_mqtt/` - MQTT integration
|
||||
- `test_packet_history/` - Packet history tracking
|
||||
- `test_position_precision/` - Position precision helpers
|
||||
- `test_radio/` - Radio interface
|
||||
- `test_serial/` - Serial communication
|
||||
- `test_traffic_management/` - Traffic management
|
||||
- `test_transmit_history/` - Retransmission tracking
|
||||
- `test_type_conversions/` - NodeDB v25 type conversion (bitfield round-trips, NodeInfoLite)
|
||||
- `test_utf8/` - UTF-8 utilities
|
||||
- `test_atak/` - ATAK integration
|
||||
- `test_default/` - Default configuration
|
||||
- `test_http_content_handler/` - HTTP handling
|
||||
- `test_serial/` - Serial communication
|
||||
|
||||
Run command (preferred — avoids pipe-buffering and the Ubuntu externally-managed-environment error):
|
||||
|
||||
```bash
|
||||
~/.platformio/penv/bin/python -m platformio test -e native -f test_your_suite > /tmp/test_out.txt 2>&1
|
||||
grep -E 'error:|PASS|FAIL|succeeded|failed' /tmp/test_out.txt
|
||||
tail -15 /tmp/test_out.txt
|
||||
```
|
||||
|
||||
Do **not** use `pio test … | tail -N` — it discards build errors and shows stale cached results. Do **not** use `pio test … | grep` — line-buffering makes the terminal appear hung until the process exits. Redirect to a file first, then grep.
|
||||
Run with: `pio test -e native`
|
||||
|
||||
Simulation testing: `bin/test-simulator.sh`
|
||||
|
||||
|
||||
@@ -4,14 +4,9 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
# trunk-ignore(checkov/CKV_GHA_7)
|
||||
target:
|
||||
type: string
|
||||
required: false
|
||||
description: Choose the target board, e.g. nrf52_promicro_diy_tcxo. If blank, will find available targets.
|
||||
arch:
|
||||
type: choice
|
||||
options:
|
||||
- all
|
||||
- esp32
|
||||
- esp32s3
|
||||
- esp32c3
|
||||
@@ -20,18 +15,32 @@ on:
|
||||
- rp2040
|
||||
- rp2350
|
||||
- stm32
|
||||
description: Choose an arch to limit the search, or 'all' to search all architectures.
|
||||
default: all
|
||||
target:
|
||||
type: string
|
||||
required: false
|
||||
description: Choose the target board, e.g. nrf52_promicro_diy_tcxo. If blank, will find available targets.
|
||||
# find-target:
|
||||
# type: boolean
|
||||
# default: true
|
||||
# description: 'Find the available targets'
|
||||
|
||||
permissions: read-all
|
||||
|
||||
jobs:
|
||||
find-targets:
|
||||
if: ${{ inputs.target == '' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
arch:
|
||||
- all
|
||||
- esp32
|
||||
- esp32s3
|
||||
- esp32c3
|
||||
- esp32c6
|
||||
- nrf52840
|
||||
- rp2040
|
||||
- rp2350
|
||||
- stm32
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
@@ -42,37 +51,14 @@ jobs:
|
||||
- run: pip install -U platformio
|
||||
- name: Generate matrix
|
||||
id: jsonStep
|
||||
env:
|
||||
BUILDTARGET: ${{ inputs.target }}
|
||||
MATRIXARCH: ${{ inputs.arch }}
|
||||
run: |
|
||||
TARGETS=$(./bin/generate_ci_matrix.py ${{matrix.arch}} --level extra)
|
||||
if [ "$BUILDTARGET" = "" ]; then
|
||||
echo "Name: $GITHUB_REF_NAME" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Base: $GITHUB_BASE_REF" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Arch: $MATRIXARCH" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Ref: $GITHUB_REF" >> $GITHUB_STEP_SUMMARY
|
||||
echo "## 🎯 The following target boards are available to build:" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Platform | Board |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| -------- | ----- |" >> $GITHUB_STEP_SUMMARY
|
||||
echo $TARGETS | jq -r 'sort_by(.board) | sort_by(.platform) |.[] | "| " + .platform + " | " + .board + " |" ' >> $GITHUB_STEP_SUMMARY
|
||||
else
|
||||
echo "We build this one:" >> $GITHUB_STEP_SUMMARY
|
||||
ARCH=$(echo "$TARGETS" | jq --arg BUILDTARGET "$BUILDTARGET" -r '.[] | select(.board==$BUILDTARGET) | .platform')
|
||||
echo "| Platform | Board |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| -------- | ----- |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| $ARCH | "$BUILDTARGET" |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
if [[ "$ARCH" == "" ]]; then
|
||||
echo "## ❌ Error: Target "$BUILDTARGET" not found!" >> $GITHUB_STEP_SUMMARY
|
||||
else
|
||||
echo "## ✅ Target "$BUILDTARGET" found, proceeding to build." >> $GITHUB_STEP_SUMMARY
|
||||
fi
|
||||
echo "You may need to refresh this page to make the built firmware appear below." >> $GITHUB_STEP_SUMMARY
|
||||
echo "arch=$ARCH" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
outputs:
|
||||
arch: ${{ steps.jsonStep.outputs.arch }}
|
||||
echo "Name: $GITHUB_REF_NAME" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Base: $GITHUB_BASE_REF" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Arch: ${{matrix.arch}}" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Ref: $GITHUB_REF" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Targets:" >> $GITHUB_STEP_SUMMARY
|
||||
echo $TARGETS | jq -r 'sort_by(.board) |.[] | "- " + .board' >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
version:
|
||||
if: ${{ inputs.target != '' }}
|
||||
@@ -92,12 +78,12 @@ jobs:
|
||||
|
||||
build:
|
||||
if: ${{ inputs.target != '' && inputs.arch != 'native' }}
|
||||
needs: [version, find-targets]
|
||||
needs: [version]
|
||||
uses: ./.github/workflows/build_firmware.yml
|
||||
with:
|
||||
version: ${{ needs.version.outputs.long }}
|
||||
pio_env: ${{ inputs.target }}
|
||||
platform: ${{ needs.find-targets.outputs.arch }}
|
||||
platform: ${{ inputs.arch }}
|
||||
|
||||
gather-artifacts:
|
||||
permissions:
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
name: Post Firmware Size Comment
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [CI]
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
pull-requests: write
|
||||
actions: read
|
||||
|
||||
jobs:
|
||||
post-size-comment:
|
||||
if: >
|
||||
github.event.workflow_run.event == 'pull_request' &&
|
||||
github.event.workflow_run.conclusion != 'cancelled' &&
|
||||
github.repository == 'meshtastic/firmware'
|
||||
continue-on-error: true
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Download size report
|
||||
id: download
|
||||
uses: actions/download-artifact@v8
|
||||
continue-on-error: true
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
run-id: ${{ github.event.workflow_run.id }}
|
||||
name: size-report
|
||||
path: ./
|
||||
|
||||
- name: Post or update PR comment
|
||||
if: steps.download.outcome == 'success'
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const marker = '<!-- firmware-size-report -->';
|
||||
const body = fs.readFileSync('./size-report.md', 'utf8');
|
||||
const prNumber = parseInt(fs.readFileSync('./pr-number.txt', 'utf8').trim(), 10);
|
||||
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: prNumber,
|
||||
});
|
||||
|
||||
const existing = comments.find(c => c.body.includes(marker));
|
||||
if (existing) {
|
||||
await github.rest.issues.updateComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
body,
|
||||
});
|
||||
} else {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: prNumber,
|
||||
body,
|
||||
});
|
||||
}
|
||||
@@ -1,187 +0,0 @@
|
||||
name: Post Web Flasher Link Comment
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [CI]
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
pull-requests: write
|
||||
actions: read
|
||||
|
||||
jobs:
|
||||
post-flasher-link:
|
||||
if: >
|
||||
github.event.workflow_run.event == 'pull_request' &&
|
||||
github.event.workflow_run.conclusion != 'cancelled' &&
|
||||
github.repository == 'meshtastic/firmware'
|
||||
continue-on-error: true
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# Per-board manifests carry the firmware's own metadata (activelySupported,
|
||||
# displayName, ...) generated from each target's custom_meshtastic_* config.
|
||||
- name: Download board manifests
|
||||
uses: actions/download-artifact@v8
|
||||
continue-on-error: true
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
run-id: ${{ github.event.workflow_run.id }}
|
||||
pattern: manifest-*
|
||||
path: ./manifests
|
||||
merge-multiple: true
|
||||
|
||||
- name: Post or update web flasher link comment
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const marker = '<!-- web-flasher-link -->';
|
||||
const run = context.payload.workflow_run;
|
||||
const { owner, repo } = context.repo;
|
||||
|
||||
// Resolve the PR by matching the run's head SHA against the repo's open
|
||||
// PRs. workflow_run.pull_requests is empty for fork PRs, and
|
||||
// listPullRequestsAssociatedWithCommit won't return an open fork PR by
|
||||
// its head commit — but pulls.list includes fork PRs. Matching on head
|
||||
// SHA also enforces that the run is for the PR's current commit, so stale
|
||||
// re-runs of an outdated commit won't match.
|
||||
const openPrs = await github.paginate(github.rest.pulls.list, {
|
||||
owner, repo, state: 'open', per_page: 100,
|
||||
});
|
||||
const pr = openPrs.find((p) => p.head.sha === run.head_sha);
|
||||
if (!pr) {
|
||||
core.info(`No open pull request matches commit ${run.head_sha}; skipping.`);
|
||||
return;
|
||||
}
|
||||
const prNumber = pr.number;
|
||||
|
||||
// Restrict to trusted authors. NOTE: author_association is computed for
|
||||
// the GITHUB_TOKEN, which cannot see *private/concealed* org memberships —
|
||||
// those members come back as CONTRIBUTOR, not MEMBER. So gating on MEMBER
|
||||
// alone silently excludes most maintainers. We allow the trusted set the
|
||||
// token can actually identify (members, collaborators, and anyone with a
|
||||
// previously merged PR). For strict members-only you'd need an org-read
|
||||
// App/PAT token to call orgs.checkMembershipForUser.
|
||||
const allowedAssociations = ['OWNER', 'MEMBER', 'COLLABORATOR', 'CONTRIBUTOR'];
|
||||
if (!allowedAssociations.includes(pr.author_association)) {
|
||||
core.info(`Author association ${pr.author_association} is not trusted; skipping.`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Require at least one per-arch firmware artifact from gather-artifacts
|
||||
const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, {
|
||||
owner, repo, run_id: run.id, per_page: 100,
|
||||
});
|
||||
const archRe = /^firmware-(esp32|esp32s3|esp32c3|esp32c6|nrf52840|rp2040|rp2350|stm32)-(\d+\.\d+\.\d+\.[0-9a-f]+)$/;
|
||||
const archArtifacts = artifacts.filter((a) => archRe.test(a.name) && !a.expired);
|
||||
if (archArtifacts.length === 0) {
|
||||
core.info('No per-arch firmware artifacts found; skipping.');
|
||||
return;
|
||||
}
|
||||
|
||||
const version = archRe.exec(archArtifacts[0].name)[2];
|
||||
const expiresAt = archArtifacts[0].expires_at
|
||||
? new Date(archArtifacts[0].expires_at).toISOString().slice(0, 10)
|
||||
: null;
|
||||
|
||||
// Read each built board's manifest (.mt.json). activelySupported,
|
||||
// displayName and architecture come straight from the board's
|
||||
// custom_meshtastic_* platformio config, so the list is in sync with
|
||||
// the firmware itself — no external device database needed.
|
||||
const fs = require('fs');
|
||||
let boards = [];
|
||||
try {
|
||||
boards = fs.readdirSync('./manifests')
|
||||
.filter((f) => f.endsWith('.mt.json'))
|
||||
.map((f) => {
|
||||
try { return JSON.parse(fs.readFileSync(`./manifests/${f}`, 'utf8')); }
|
||||
catch { return null; }
|
||||
})
|
||||
.filter((m) => m && m.activelySupported === true && m.platformioTarget)
|
||||
.map((m) => ({
|
||||
board: m.platformioTarget,
|
||||
platform: m.architecture || '',
|
||||
// displayName is maintainer-authored text; escape table-breaking pipes
|
||||
displayName: String(m.displayName || m.platformioTarget).replace(/\|/g, '\\|'),
|
||||
image: Array.isArray(m.images) && m.images[0] ? String(m.images[0]) : '',
|
||||
}))
|
||||
.sort((a, b) => a.board.localeCompare(b.board));
|
||||
} catch (e) {
|
||||
core.warning(`Could not read board manifests: ${e.message}`);
|
||||
}
|
||||
|
||||
const flasherUrl = `https://flasher.meshtastic.org/?pr=${prNumber}`;
|
||||
// Device illustrations are served by the flasher from the same image
|
||||
// names the manifest declares (custom_meshtastic_images). The flasher
|
||||
// serves its SPA shell (HTML, 200) for unknown paths, so confirm each
|
||||
// image really resolves to an image before linking it.
|
||||
const imageBase = 'https://flasher.meshtastic.org/img/devices/';
|
||||
await Promise.all(boards.map(async (b) => {
|
||||
if (!b.image) return;
|
||||
try {
|
||||
const res = await fetch(`${imageBase}${encodeURIComponent(b.image)}`);
|
||||
const type = res.headers.get('content-type') || '';
|
||||
if (!res.ok || !type.startsWith('image/')) b.image = '';
|
||||
} catch { b.image = ''; }
|
||||
}));
|
||||
|
||||
const boardLines = boards
|
||||
.map((b) => {
|
||||
const img = b.image ? `<img src="${imageBase}${encodeURIComponent(b.image)}" alt="" height="34">` : '';
|
||||
return `| ${img} | ${b.displayName} | [\`${b.board}\`](${flasherUrl}&device=${encodeURIComponent(b.board)}) | ${b.platform} |`;
|
||||
})
|
||||
.join('\n');
|
||||
|
||||
// Shields.io badges. Only non-user-controlled, charset-constrained values
|
||||
// (version, commit sha, counts, dates) go into badge URLs — never board
|
||||
// names or the PR title — so the rendered comment cannot be spoofed.
|
||||
const shieldText = (s) =>
|
||||
encodeURIComponent(String(s).replace(/-/g, '--').replace(/_/g, '__').replace(/ /g, '_'));
|
||||
const shield = (label, message, color) =>
|
||||
`https://img.shields.io/badge/${shieldText(label)}-${shieldText(message)}-${color}`;
|
||||
const buttonUrl =
|
||||
`https://img.shields.io/badge/${shieldText('Flash this PR in the Web Flasher')}-2C2D3C?style=for-the-badge`;
|
||||
const badges = [
|
||||
`})`,
|
||||
`, '2C2D3C')})`,
|
||||
`})`,
|
||||
];
|
||||
if (expiresAt) badges.push(`})`);
|
||||
|
||||
// Only render the board table when there are supported boards to list
|
||||
const boardTable = boards.length > 0 ? [
|
||||
`<details><summary>Supported boards built by this PR (${boards.length})</summary>`,
|
||||
'',
|
||||
'| | Device | Board | Platform |',
|
||||
'| --- | --- | --- | --- |',
|
||||
boardLines,
|
||||
'',
|
||||
'</details>',
|
||||
'',
|
||||
] : [];
|
||||
|
||||
const body = [
|
||||
marker,
|
||||
'## ⚡ Try this PR in the Web Flasher',
|
||||
'',
|
||||
`[](${flasherUrl})`,
|
||||
'',
|
||||
badges.join(' '),
|
||||
'',
|
||||
'> [!WARNING]',
|
||||
'> This is an automated, unreviewed CI test build. Back up your device configuration',
|
||||
'> before flashing, and only flash devices you are able to recover.',
|
||||
'',
|
||||
...boardTable,
|
||||
`*Build artifacts expire${expiresAt ? ` on ${expiresAt}` : ' after 30 days'}. Updated for \`${run.head_sha.slice(0, 7)}\`.*`,
|
||||
].join('\n');
|
||||
|
||||
// Sticky comment: update in place when the marker is found
|
||||
const comments = await github.paginate(github.rest.issues.listComments, {
|
||||
owner, repo, issue_number: prNumber, per_page: 100,
|
||||
});
|
||||
const existing = comments.find((c) => c.body?.includes(marker));
|
||||
if (existing) {
|
||||
await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body });
|
||||
} else {
|
||||
await github.rest.issues.createComment({ owner, repo, issue_number: prNumber, body });
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
name: Post Web Flasher Build Placeholder
|
||||
|
||||
# Drops an immediate "build in progress" comment when a PR opens, so the web
|
||||
# flasher entry shows up right away. The real CI-driven workflow
|
||||
# (flasher-link-comment.yml) later replaces it in place via the shared marker.
|
||||
#
|
||||
# SECURITY: this uses pull_request_target (write token, runs for fork PRs) but is
|
||||
# safe because it never checks out or runs PR code and posts a fully static body
|
||||
# — no PR title, branch name, or other untrusted input is used anywhere.
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, reopened]
|
||||
|
||||
permissions:
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
post-placeholder:
|
||||
if: github.repository == 'meshtastic/firmware'
|
||||
continue-on-error: true
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Post web flasher build-in-progress placeholder
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const marker = '<!-- web-flasher-link -->';
|
||||
const { owner, repo } = context.repo;
|
||||
const pr = context.payload.pull_request;
|
||||
|
||||
// Trusted authors only (matches the real workflow). author_association
|
||||
// can't reflect private org membership for the token, so concealed
|
||||
// members appear as CONTRIBUTOR — include it, or maintainers are excluded.
|
||||
const allowedAssociations = ['OWNER', 'MEMBER', 'COLLABORATOR', 'CONTRIBUTOR'];
|
||||
if (!allowedAssociations.includes(pr.author_association)) {
|
||||
core.info(`Author association ${pr.author_association} is not trusted; skipping.`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Only seed a placeholder when no flasher comment exists yet — never
|
||||
// overwrite a real (or existing placeholder) comment.
|
||||
const comments = await github.paginate(github.rest.issues.listComments, {
|
||||
owner, repo, issue_number: pr.number, per_page: 100,
|
||||
});
|
||||
if (comments.some((c) => c.body?.includes(marker))) {
|
||||
core.info('Flasher comment already exists; nothing to do.');
|
||||
return;
|
||||
}
|
||||
|
||||
const body = [
|
||||
marker,
|
||||
'## ⚡ Try this PR in the Web Flasher',
|
||||
'',
|
||||
'> [!NOTE]',
|
||||
'> Building this pull request… the flash button, badges and supported-board',
|
||||
'> list will appear here automatically once CI finishes.',
|
||||
].join('\n');
|
||||
|
||||
await github.rest.issues.createComment({
|
||||
owner, repo, issue_number: pr.number, body,
|
||||
});
|
||||
@@ -82,9 +82,8 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
check: ${{ fromJson(needs.setup.outputs.check) }}
|
||||
# Runs on GitHub-hosted runners so checks don't compete with builds for the
|
||||
# self-hosted 'arctastic' pool (which builds use).
|
||||
runs-on: ubuntu-latest
|
||||
# Use 'arctastic' self-hosted runner pool when checking in the main repo
|
||||
runs-on: ${{ github.repository_owner == 'meshtastic' && 'arctastic' || 'ubuntu-latest' }}
|
||||
if: ${{ github.event_name != 'workflow_dispatch' && github.repository == 'meshtastic/firmware' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
@@ -246,126 +245,47 @@ jobs:
|
||||
path: ./*.elf
|
||||
retention-days: 30
|
||||
|
||||
firmware-size-report:
|
||||
shame:
|
||||
if: github.repository == 'meshtastic/firmware'
|
||||
continue-on-error: true
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Download current manifests
|
||||
if: github.event_name == 'pull_request'
|
||||
with:
|
||||
filter: blob:none # means we download all the git history but none of the commit (except ones with checkout like the head)
|
||||
fetch-depth: 0
|
||||
- name: Download the current manifests
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
path: ./manifests/
|
||||
path: ./manifests-new/
|
||||
pattern: manifest-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Collect current firmware sizes
|
||||
run: python3 bin/collect_sizes.py ./manifests/ ./current-sizes.json
|
||||
|
||||
- name: Upload size report artifact
|
||||
- name: Upload combined manifests for later commit and global stats crunching.
|
||||
uses: actions/upload-artifact@v7
|
||||
id: upload-manifest
|
||||
with:
|
||||
name: firmware-sizes-${{ github.sha }}
|
||||
name: manifests-${{ github.sha }}
|
||||
overwrite: true
|
||||
path: ./current-sizes.json
|
||||
retention-days: 90
|
||||
|
||||
- name: Download baseline sizes from develop
|
||||
path: manifests-new/*.mt.json
|
||||
- name: Find the merge base
|
||||
if: github.event_name == 'pull_request'
|
||||
continue-on-error: true
|
||||
id: baseline-develop
|
||||
run: echo "MERGE_BASE=$(git merge-base "origin/$base" "$head")" >> $GITHUB_ENV
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
RUN_ID=$(gh run list -R "${{ github.repository }}" \
|
||||
--workflow CI --branch develop --status success \
|
||||
--limit 1 --json databaseId --jq '.[0].databaseId // empty')
|
||||
if [ -n "$RUN_ID" ]; then
|
||||
ARTIFACT_NAME=$(gh api "repos/${{ github.repository }}/actions/runs/${RUN_ID}/artifacts" \
|
||||
--jq '.artifacts[] | select(.name | startswith("firmware-sizes-")) | select(.expired == false) | .name' | head -1)
|
||||
if [ -n "$ARTIFACT_NAME" ]; then
|
||||
gh run download "$RUN_ID" -R "${{ github.repository }}" \
|
||||
--name "$ARTIFACT_NAME" --dir ./baseline-develop/
|
||||
cp "./baseline-develop/current-sizes.json" ./develop-sizes.json
|
||||
echo "found=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "found=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
else
|
||||
echo "found=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Download baseline sizes from master
|
||||
if: github.event_name == 'pull_request'
|
||||
continue-on-error: true
|
||||
id: baseline-master
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
RUN_ID=$(gh run list -R "${{ github.repository }}" \
|
||||
--workflow CI --branch master --status success \
|
||||
--limit 1 --json databaseId --jq '.[0].databaseId // empty')
|
||||
if [ -n "$RUN_ID" ]; then
|
||||
ARTIFACT_NAME=$(gh api "repos/${{ github.repository }}/actions/runs/${RUN_ID}/artifacts" \
|
||||
--jq '.artifacts[] | select(.name | startswith("firmware-sizes-")) | select(.expired == false) | .name' | head -1)
|
||||
if [ -n "$ARTIFACT_NAME" ]; then
|
||||
gh run download "$RUN_ID" -R "${{ github.repository }}" \
|
||||
--name "$ARTIFACT_NAME" --dir ./baseline-master/
|
||||
cp "./baseline-master/current-sizes.json" ./master-sizes.json
|
||||
echo "found=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "found=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
else
|
||||
echo "found=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Generate size comparison report
|
||||
if: github.event_name == 'pull_request'
|
||||
id: report
|
||||
run: |
|
||||
ARGS="./current-sizes.json"
|
||||
if [ -f ./develop-sizes.json ]; then
|
||||
ARGS="$ARGS --baseline develop:./develop-sizes.json"
|
||||
fi
|
||||
if [ -f ./master-sizes.json ]; then
|
||||
ARGS="$ARGS --baseline master:./master-sizes.json"
|
||||
fi
|
||||
REPORT=$(python3 bin/size_report.py $ARGS)
|
||||
if [ -z "$REPORT" ]; then
|
||||
echo "has_report=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "has_report=true" >> "$GITHUB_OUTPUT"
|
||||
{
|
||||
echo '<!-- firmware-size-report -->'
|
||||
echo '# Firmware Size Report'
|
||||
echo ''
|
||||
echo "$REPORT"
|
||||
echo ''
|
||||
echo '---'
|
||||
echo "*Updated for ${{ github.sha }}*"
|
||||
} > ./size-report.md
|
||||
cat ./size-report.md >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
|
||||
- name: Save PR number
|
||||
if: github.event_name == 'pull_request' && steps.report.outputs.has_report == 'true'
|
||||
run: echo "${{ github.event.pull_request.number }}" > ./pr-number.txt
|
||||
|
||||
- name: Upload size report
|
||||
if: github.event_name == 'pull_request' && steps.report.outputs.has_report == 'true'
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: size-report
|
||||
path: |
|
||||
./size-report.md
|
||||
./pr-number.txt
|
||||
retention-days: 5
|
||||
base: ${{ github.base_ref }}
|
||||
head: ${{ github.sha }}
|
||||
# Currently broken (for-loop through EVERY artifact -- rate limiting)
|
||||
# - name: Download the old manifests
|
||||
# if: github.event_name == 'pull_request'
|
||||
# run: gh run download -R "$repo" --name "manifests-$merge_base" --dir manifest-old/
|
||||
# env:
|
||||
# GH_TOKEN: ${{ github.token }}
|
||||
# merge_base: ${{ env.MERGE_BASE }}
|
||||
# repo: ${{ github.repository }}
|
||||
# - name: Do scan and post comment
|
||||
# if: github.event_name == 'pull_request'
|
||||
# run: python3 bin/shame.py ${{ github.event.pull_request.number }} manifests-old/ manifests-new/
|
||||
|
||||
release-artifacts:
|
||||
permissions: # Needed for 'gh release upload'.
|
||||
@@ -413,7 +333,6 @@ jobs:
|
||||
prerelease: true
|
||||
name: Meshtastic Firmware ${{ needs.version.outputs.long }} Alpha
|
||||
tag_name: v${{ needs.version.outputs.long }}
|
||||
target_commitish: ${{ github.sha }}
|
||||
body: ${{ steps.release_notes.outputs.notes }}
|
||||
|
||||
- name: Download source deb
|
||||
|
||||
@@ -37,33 +37,13 @@ jobs:
|
||||
sed -i -e "s#${PWD}#.#" coverage_base.info # Make paths relative.
|
||||
|
||||
- name: Integration test
|
||||
# Cap the whole step: if the simulator ever fails to exit (e.g. the
|
||||
# exit_simulator admin path regresses again) the job must fail fast,
|
||||
# not run to GitHub's 6-hour limit.
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
.pio/build/coverage/meshtasticd -s &
|
||||
PID=$!
|
||||
trap 'kill "$PID" 2>/dev/null || true' EXIT
|
||||
timeout 20 bash -c "until ls -al /proc/$PID/fd | grep socket; do sleep 1; done"
|
||||
echo "Simulator started, launching python test..."
|
||||
python3 -c 'from meshtastic.test import testSimulator; testSimulator()'
|
||||
# The Python harness sends exit_simulator and exits; the simulator is
|
||||
# expected to terminate on its own. Give it a moment, then verify.
|
||||
# If it is still alive the exit handshake is broken — fail loudly and
|
||||
# do NOT fall through to `wait`, which would otherwise block until the
|
||||
# job's hard timeout.
|
||||
for i in $(seq 1 10); do
|
||||
kill -0 "$PID" 2>/dev/null || break
|
||||
sleep 1
|
||||
done
|
||||
if kill -0 "$PID" 2>/dev/null; then
|
||||
echo "::error title=Simulator did not exit::meshtasticd ignored exit_simulator and is still running after the integration test. The exit_simulator admin path is broken (see AdminModule::handleReceivedProtobuf, ARCH_PORTDUINO bypass). Killing it to avoid a 6-hour CI overrun."
|
||||
kill -9 "$PID" 2>/dev/null || true
|
||||
wait "$PID" 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
wait "$PID" 2>/dev/null || true
|
||||
wait
|
||||
|
||||
- name: Capture coverage information
|
||||
if: always() # run this step even if previous step failed
|
||||
@@ -106,13 +86,7 @@ jobs:
|
||||
run: sed -i 's/-DBUILD_EPOCH=$UNIX_TIME/#-DBUILD_EPOCH=$UNIX_TIME/' platformio.ini
|
||||
|
||||
- name: PlatformIO Tests
|
||||
run: |
|
||||
set -o pipefail
|
||||
# Filter out SKIPPED summary rows for hardware variants that can't run on the
|
||||
# native host. They flood the log and make it harder to spot real failures.
|
||||
# The JUnit XML is written directly to testreport.xml before the pipe, so
|
||||
# the test artifact is unaffected.
|
||||
platformio test -e coverage -v --junit-output-path testreport.xml 2>&1 | grep -v "[[:space:]]SKIPPED$"
|
||||
run: platformio test -e coverage -v --junit-output-path testreport.xml
|
||||
|
||||
- name: Save test results
|
||||
if: always() # run this step even if previous step failed
|
||||
@@ -161,14 +135,6 @@ jobs:
|
||||
name: platformio-test-report-${{ steps.version.outputs.long }}
|
||||
merge-multiple: true
|
||||
|
||||
- name: Drop no-status testsuites from the report
|
||||
# PlatformIO emits a self-closing <testsuite tests="0"/> row for every test_* dir
|
||||
# crossed with every hardware variant it cannot run on the native host (~4900 rows).
|
||||
# They carry no pass/fail/skip status and bury the suites that actually ran. Strip
|
||||
# them so the Test Report lists only suites with a real status. Only the copy the
|
||||
# reporter renders is trimmed; the uploaded artifact keeps the full XML.
|
||||
run: sed -i -E 's#<testsuite [^>]*tests="0"[^>]*/>##g' testreport.xml
|
||||
|
||||
- name: Test Report
|
||||
uses: dorny/test-reporter@v3.0.0
|
||||
with:
|
||||
|
||||
@@ -16,18 +16,13 @@ jobs:
|
||||
submodules: true
|
||||
|
||||
- name: Update submodule
|
||||
if: ${{ github.ref_name == 'master' || github.ref_name == 'develop' }}
|
||||
working-directory: protobufs
|
||||
env:
|
||||
# Use the branch that triggered the workflow as the protobuf branch.
|
||||
GIT_BRANCH: ${{ github.ref_name }}
|
||||
if: ${{ github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop' }}
|
||||
run: |
|
||||
git fetch --prune origin $GIT_BRANCH
|
||||
git checkout origin/$GIT_BRANCH
|
||||
git submodule update --remote protobufs
|
||||
|
||||
- name: Download nanopb
|
||||
run: |
|
||||
wget https://github.com/nanopb/nanopb/releases/download/nanopb-0.4.9.1/nanopb-0.4.9.1-linux-x86.tar.gz
|
||||
wget https://jpa.kapsi.fi/nanopb/download/nanopb-0.4.9.1-linux-x86.tar.gz
|
||||
tar xvzf nanopb-0.4.9.1-linux-x86.tar.gz
|
||||
mv nanopb-0.4.9.1-linux-x86 nanopb-0.4.9
|
||||
|
||||
@@ -38,7 +33,7 @@ jobs:
|
||||
- name: Create pull request
|
||||
uses: peter-evans/create-pull-request@v8
|
||||
with:
|
||||
branch: create-pull-request/update-protobufs-${{ github.ref_name }}
|
||||
branch: create-pull-request/update-protobufs
|
||||
labels: submodules
|
||||
title: Update protobufs and classes
|
||||
commit-message: Update protobufs
|
||||
|
||||
+5
-15
@@ -1,8 +1,8 @@
|
||||
/.pio
|
||||
/pio
|
||||
/pio.tar
|
||||
/web
|
||||
/web.tar
|
||||
.pio
|
||||
pio
|
||||
pio.tar
|
||||
web
|
||||
web.tar
|
||||
|
||||
# ignore vscode IDE settings files
|
||||
.vscode/*
|
||||
@@ -47,10 +47,6 @@ data/boot/logo.*
|
||||
managed_components/*
|
||||
arduino-lib-builder*
|
||||
dependencies.lock
|
||||
|
||||
# JLink / RTT debug artifacts (nRF SoCs)
|
||||
flash.jlink
|
||||
rtt_*.txt
|
||||
idf_component.yml
|
||||
CMakeLists.txt
|
||||
/sdkconfig.*
|
||||
@@ -60,9 +56,3 @@ CMakeLists.txt
|
||||
.python3
|
||||
.claude/scheduled_tasks.lock
|
||||
userPrefs.jsonc.mcp-session-bak
|
||||
|
||||
# Fake-NodeDB fixture pipeline (bin/regen-fake-nodedbs.sh)
|
||||
# JSONL seeds are committed (test/fixtures/nodedb/seed_v25_*.jsonl);
|
||||
# compiled .proto outputs are ephemeral build artifacts.
|
||||
build/fixtures/
|
||||
bin/_generated/
|
||||
|
||||
+5
-12
@@ -4,19 +4,19 @@ cli:
|
||||
plugins:
|
||||
sources:
|
||||
- id: trunk
|
||||
ref: v1.10.0
|
||||
ref: v1.9.0
|
||||
uri: https://github.com/trunk-io/plugins
|
||||
lint:
|
||||
enabled:
|
||||
- checkov@3.2.529
|
||||
- checkov@3.2.528
|
||||
- renovate@43.150.0
|
||||
- prettier@3.8.3
|
||||
- trufflehog@3.95.3
|
||||
- trufflehog@3.95.2
|
||||
- yamllint@1.38.0
|
||||
- bandit@1.9.4
|
||||
- trivy@0.70.0
|
||||
- taplo@0.10.0
|
||||
- ruff@0.15.13
|
||||
- ruff@0.15.12
|
||||
- isort@8.0.1
|
||||
- markdownlint@0.48.0
|
||||
- oxipng@10.1.1
|
||||
@@ -26,7 +26,7 @@ lint:
|
||||
- hadolint@2.14.0
|
||||
- shfmt@3.6.0
|
||||
- shellcheck@0.11.0
|
||||
- black@26.5.1
|
||||
- black@26.3.1
|
||||
- git-diff-check
|
||||
- gitleaks@8.30.1
|
||||
- clang-format@16.0.3
|
||||
@@ -34,13 +34,6 @@ lint:
|
||||
- linters: [ALL]
|
||||
paths:
|
||||
- bin/**
|
||||
# Fake-NodeDB fixture JSONL files contain deterministic synthetic
|
||||
# public_key_hex (64-char hex) values that gitleaks misidentifies as
|
||||
# generic-api-key. These are not secrets — they're test fixtures
|
||||
# produced by bin/gen-fake-nodedb-seed.py with a fixed RNG seed.
|
||||
- linters: [gitleaks]
|
||||
paths:
|
||||
- test/fixtures/nodedb/seed_v25_*.jsonl
|
||||
runtimes:
|
||||
enabled:
|
||||
- python@3.14.4
|
||||
|
||||
Vendored
+4
-4
@@ -1,10 +1,10 @@
|
||||
{
|
||||
// See http://go.microsoft.com/fwlink/?LinkId=827846
|
||||
// for the documentation about the extensions.json format
|
||||
"recommendations": [
|
||||
"Jason2866.esp-decoder",
|
||||
"pioarduino.pioarduino-ide"
|
||||
"platformio.platformio-ide"
|
||||
],
|
||||
"unwantedRecommendations": [
|
||||
"ms-vscode.cpptools-extension-pack",
|
||||
"platformio.platformio-ide"
|
||||
"ms-vscode.cpptools-extension-pack"
|
||||
]
|
||||
}
|
||||
|
||||
Vendored
-41
@@ -10,47 +10,6 @@
|
||||
"isDefault": true
|
||||
},
|
||||
"label": "PlatformIO: Build"
|
||||
},
|
||||
{
|
||||
"label": "FleetSuite",
|
||||
"detail": "Build the SPA if needed and open the FleetSuite desktop window",
|
||||
"type": "shell",
|
||||
"command": "./scripts/fleetsuite.sh",
|
||||
"options": { "cwd": "${workspaceFolder}/mcp-server" },
|
||||
"problemMatcher": [],
|
||||
"presentation": {
|
||||
"reveal": "always",
|
||||
"panel": "dedicated",
|
||||
"clear": true
|
||||
},
|
||||
"group": "none"
|
||||
},
|
||||
{
|
||||
"label": "FleetSuite: Dev (HMR)",
|
||||
"detail": "Run the backend + Vite dev server with hot-reload",
|
||||
"type": "shell",
|
||||
"command": "./scripts/fleetsuite.sh --dev",
|
||||
"options": { "cwd": "${workspaceFolder}/mcp-server" },
|
||||
"problemMatcher": [],
|
||||
"presentation": {
|
||||
"reveal": "always",
|
||||
"panel": "dedicated",
|
||||
"clear": true
|
||||
},
|
||||
"isBackground": true
|
||||
},
|
||||
{
|
||||
"label": "FleetSuite: Rebuild + Run",
|
||||
"detail": "Force a fresh SPA build, then open the window",
|
||||
"type": "shell",
|
||||
"command": "./scripts/fleetsuite.sh --rebuild",
|
||||
"options": { "cwd": "${workspaceFolder}/mcp-server" },
|
||||
"problemMatcher": [],
|
||||
"presentation": {
|
||||
"reveal": "always",
|
||||
"panel": "dedicated",
|
||||
"clear": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -10,18 +10,18 @@ This file (`AGENTS.md`) is a short pointer + quick reference for agents that don
|
||||
|
||||
## Quick command reference
|
||||
|
||||
| Action | Command |
|
||||
| -------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| Build a firmware variant | `pio run -e <env>` (e.g. `pio run -e rak4631`, `pio run -e heltec-v3`) |
|
||||
| Build native macOS host binary | `pio run -e native-macos` (Homebrew prereqs + CH341 LoRa setup in `variants/native/portduino/platformio.ini`) |
|
||||
| Clean + rebuild | `pio run -e <env> -t clean && pio run -e <env>` |
|
||||
| Flash a device | `pio run -e <env> -t upload --upload-port <port>` (or use the `pio_flash` MCP tool) |
|
||||
| Run firmware unit tests (native) | `~/.platformio/penv/bin/python -m platformio test -e native > /tmp/test_out.txt 2>&1` then `grep -E 'error:\|PASS\|FAIL\|succeeded\|failed' /tmp/test_out.txt` (redirect first — piping causes line-buffering) |
|
||||
| Run MCP hardware tests | `./mcp-server/run-tests.sh` |
|
||||
| Live TUI test runner | `mcp-server/.venv/bin/meshtastic-mcp-test-tui` |
|
||||
| Format before commit | `trunk fmt` |
|
||||
| Regenerate protobuf bindings | `bin/regen-protos.sh` |
|
||||
| Generate CI matrix | `./bin/generate_ci_matrix.py all [--level pr]` |
|
||||
| Action | Command |
|
||||
| -------------------------------- | ------------------------------------------------------------------------------------------------------------- |
|
||||
| Build a firmware variant | `pio run -e <env>` (e.g. `pio run -e rak4631`, `pio run -e heltec-v3`) |
|
||||
| Build native macOS host binary | `pio run -e native-macos` (Homebrew prereqs + CH341 LoRa setup in `variants/native/portduino/platformio.ini`) |
|
||||
| Clean + rebuild | `pio run -e <env> -t clean && pio run -e <env>` |
|
||||
| Flash a device | `pio run -e <env> -t upload --upload-port <port>` (or use the `pio_flash` MCP tool) |
|
||||
| Run firmware unit tests (native) | `pio test -e native` |
|
||||
| Run MCP hardware tests | `./mcp-server/run-tests.sh` |
|
||||
| Live TUI test runner | `mcp-server/.venv/bin/meshtastic-mcp-test-tui` |
|
||||
| Format before commit | `trunk fmt` |
|
||||
| Regenerate protobuf bindings | `bin/regen-protos.sh` |
|
||||
| Generate CI matrix | `./bin/generate_ci_matrix.py all [--level pr]` |
|
||||
|
||||
## MCP server (device + test automation)
|
||||
|
||||
@@ -64,7 +64,7 @@ Key rotation to never trigger casually: only the **full** factory reset (`factor
|
||||
- **One MCP call per serial port at a time.** The port lock is exclusive; concurrent calls deadlock. Sequence: open → read/mutate → close, then next device.
|
||||
- **`userPrefs.jsonc` is session state during tests.** The `_session_userprefs` fixture snapshots + restores it; never edit it from inside a test.
|
||||
- **Don't speculate about firmware root causes.** When evidence doesn't support a classification, say "unknown" and list what would disambiguate.
|
||||
- **Run `trunk fmt` before proposing a commit.** The `trunk_check` CI gate will reject unformatted code. Claude Code runs it automatically via the PostToolUse hook in `.claude/settings.json`; trunk's launcher needs `curl` or `wget` to bootstrap its pinned CLI — see **Formatting & the trunk toolchain** in `.github/copilot-instructions.md` for the no-curl bootstrap procedure.
|
||||
- **Run `trunk fmt` before proposing a commit.** The `trunk_check` CI gate will reject unformatted code.
|
||||
- **`confirm=True` on destructive MCP tools is a real gate, not a formality.** Don't bypass it via auto-approve settings.
|
||||
- **Keep code comments minimal — one or two lines, max.** Comment only when the _why_ isn't obvious from the code; never restate what the next line does. No multi-paragraph block comments explaining straightforward changes. The diff and commit message carry the rationale; the code carries the behavior.
|
||||
- **Use `Throttle` for time-based rate limiting, not raw `millis()` math.** `src/mesh/Throttle.h` provides `Throttle::isWithinTimespanMs(lastMs, intervalMs)` (returns true while inside the cooldown) and `Throttle::execute(&lastMs, intervalMs, func)` (function-pointer form that updates the timestamp on fire). Use these for any "did N ms pass since X" check — raw `millis() > lastMs + N` is rollover-unsafe (breaks after ~49.7 days) and inconsistent with the rest of the codebase. The helpers compute `now - lastMs` with unsigned subtraction, which wraps correctly.
|
||||
@@ -108,7 +108,7 @@ Sequence these; don't parallelize on the same port.
|
||||
| `src/modules/` | Feature modules; `Telemetry/Sensor/` has 50+ I2C sensor drivers |
|
||||
| `variants/` | 200+ hardware variant definitions (`variant.h` + `platformio.ini` per board) |
|
||||
| `protobufs/` | `.proto` definitions; regenerate with `bin/regen-protos.sh` |
|
||||
| `test/` | Firmware unit tests (17 suites; `pio test -e native`) |
|
||||
| `test/` | Firmware unit tests (12 suites; `pio test -e native`) |
|
||||
| `mcp-server/` | Python MCP server + pytest hardware integration tests |
|
||||
| `mcp-server/tests/` | Tiered pytest suite: `unit/`, `mesh/`, `telemetry/`, `monitor/`, `recovery/`, `ui/`, `fleet/`, `admin/`, `provisioning/` |
|
||||
| `.claude/commands/` | Claude Code slash command bodies |
|
||||
|
||||
+2
-2
@@ -14,7 +14,7 @@ ENV PIP_BREAK_SYSTEM_PACKAGES=1
|
||||
RUN apt-get update && apt-get install --no-install-recommends -y \
|
||||
curl wget g++ zip git ca-certificates pkg-config \
|
||||
python3-pip python3-grpc-tools \
|
||||
libgpiod-dev libyaml-cpp-dev libjsoncpp-dev libbluetooth-dev libi2c-dev libuv1-dev \
|
||||
libgpiod-dev libyaml-cpp-dev libbluetooth-dev libi2c-dev libuv1-dev \
|
||||
libusb-1.0-0-dev libulfius-dev liborcania-dev libssl-dev \
|
||||
libx11-dev libinput-dev libxkbcommon-x11-dev libsqlite3-dev libsdl2-dev \
|
||||
&& apt-get clean && rm -rf /var/lib/apt/lists/* \
|
||||
@@ -53,7 +53,7 @@ ENV TZ=Etc/UTC
|
||||
USER root
|
||||
|
||||
RUN apt-get update && apt-get --no-install-recommends -y install \
|
||||
libc-bin libc6 libgpiod3 libyaml-cpp0.8 libjsoncpp26 libi2c0 libuv1t64 libusb-1.0-0-dev \
|
||||
libc-bin libc6 libgpiod3 libyaml-cpp0.8 libi2c0 libuv1t64 libusb-1.0-0-dev \
|
||||
liborcania2.3 libulfius2.7t64 libssl3t64 \
|
||||
libx11-6 libinput10 libxkbcommon-x11-0 libsdl2-2.0-0 \
|
||||
&& apt-get clean && rm -rf /var/lib/apt/lists/* \
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@ ENV PIP_ROOT_USER_ACTION=ignore
|
||||
# hadolint ignore=DL3008
|
||||
RUN apt-get update && apt-get install --no-install-recommends -y \
|
||||
g++ git ca-certificates pkg-config \
|
||||
libgpiod-dev libyaml-cpp-dev libjsoncpp-dev libbluetooth-dev libi2c-dev libuv1-dev \
|
||||
libgpiod-dev libyaml-cpp-dev libbluetooth-dev libi2c-dev libuv1-dev \
|
||||
libusb-1.0-0-dev libulfius-dev liborcania-dev libssl-dev \
|
||||
libx11-dev libinput-dev libxkbcommon-x11-dev libsqlite3-dev libsdl2-dev \
|
||||
&& apt-get clean && rm -rf /var/lib/apt/lists/* \
|
||||
|
||||
+2
-2
@@ -16,7 +16,7 @@ ENV PIP_BREAK_SYSTEM_PACKAGES=1
|
||||
RUN apk --no-cache add \
|
||||
bash g++ libstdc++-dev linux-headers zip git ca-certificates libbsd-dev \
|
||||
py3-pip py3-grpcio-tools \
|
||||
libgpiod-dev yaml-cpp-dev jsoncpp-dev bluez-dev \
|
||||
libgpiod-dev yaml-cpp-dev bluez-dev \
|
||||
libusb-dev i2c-tools-dev libuv-dev openssl-dev pkgconf argp-standalone \
|
||||
libx11-dev libinput-dev libxkbcommon-dev sqlite-dev sdl2-dev \
|
||||
&& rm -rf /var/cache/apk/* \
|
||||
@@ -48,7 +48,7 @@ LABEL org.opencontainers.image.title="Meshtastic" \
|
||||
USER root
|
||||
|
||||
RUN apk --no-cache add \
|
||||
shadow libstdc++ libbsd libgpiod yaml-cpp jsoncpp libusb \
|
||||
shadow libstdc++ libbsd libgpiod yaml-cpp libusb \
|
||||
i2c-tools libuv libx11 libinput libxkbcommon sdl2 \
|
||||
&& rm -rf /var/cache/apk/* \
|
||||
&& mkdir -p /var/lib/meshtasticd \
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Post-process protoc-generated Python files to live under a local namespace.
|
||||
|
||||
Called by bin/regen-py-protos.sh. Walks the generated *_pb2.py files in the
|
||||
target directory and rewrites every `meshtastic` reference (imports, dotted
|
||||
attribute access) to use the new namespace (e.g., `meshtastic_v25`).
|
||||
|
||||
Why: the .proto files declare `package meshtastic;`, so protoc emits
|
||||
`from meshtastic import mesh_pb2 as ...` lines. That would shadow the PyPI
|
||||
`meshtastic` package which other parts of the mcp-server depend on. Renaming
|
||||
to a local namespace keeps both available.
|
||||
|
||||
Usage:
|
||||
_rewrite_proto_namespace.py <generated_dir> <new_namespace>
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
|
||||
|
||||
def rewrite(dir_path: pathlib.Path, new_ns: str) -> int:
|
||||
# Standard protoc import forms:
|
||||
# from meshtastic.X_pb2 import ... (rare, for direct symbol pulls)
|
||||
# from meshtastic import X_pb2 as ... (common, the cross-file ref)
|
||||
# import meshtastic.X_pb2 (also possible)
|
||||
pattern_dotted_from = re.compile(r"^from meshtastic\.", re.MULTILINE)
|
||||
pattern_bare_from = re.compile(r"^from meshtastic import ", re.MULTILINE)
|
||||
pattern_dotted_import = re.compile(r"^import meshtastic\.", re.MULTILINE)
|
||||
|
||||
count = 0
|
||||
for p in dir_path.glob("*.py"):
|
||||
text = p.read_text(encoding="utf-8")
|
||||
new = pattern_dotted_from.sub(f"from {new_ns}.", text)
|
||||
new = pattern_bare_from.sub(f"from {new_ns} import ", new)
|
||||
new = pattern_dotted_import.sub(f"import {new_ns}.", new)
|
||||
# NOTE: we deliberately leave `meshtastic/X.proto` source-filename
|
||||
# references inside descriptor strings alone. The descriptor pool is
|
||||
# keyed by source filename (independent of Python package layout), so
|
||||
# those don't collide with the PyPI package's descriptors.
|
||||
if new != text:
|
||||
p.write_text(new, encoding="utf-8")
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) != 2:
|
||||
print("usage: _rewrite_proto_namespace.py <generated_dir> <new_namespace>", file=sys.stderr)
|
||||
return 2
|
||||
dir_path = pathlib.Path(argv[0])
|
||||
new_ns = argv[1]
|
||||
if not dir_path.is_dir():
|
||||
print(f"directory not found: {dir_path}", file=sys.stderr)
|
||||
return 2
|
||||
n = rewrite(dir_path, new_ns)
|
||||
print(f"rewrote {n} file(s) in {dir_path} → namespace {new_ns}", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
+1
-1
@@ -38,4 +38,4 @@ cp bin/device-install.* $OUTDIR/
|
||||
cp bin/device-update.* $OUTDIR/
|
||||
|
||||
echo "Copying manifest"
|
||||
cp $BUILDDIR/$basename.mt.json $OUTDIR/$basename.mt.json
|
||||
cp $BUILDDIR/$basename.mt.json $OUTDIR/$basename.mt.json || true
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""Collect firmware binary sizes from manifest (.mt.json) files into a single report."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
def collect_sizes(manifest_dir):
|
||||
"""Scan manifest_dir for .mt.json files and return {board: size_bytes} dict."""
|
||||
sizes = {}
|
||||
for fname in sorted(os.listdir(manifest_dir)):
|
||||
if not fname.endswith(".mt.json"):
|
||||
continue
|
||||
path = os.path.join(manifest_dir, fname)
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
board = data.get("platformioTarget", fname.replace(".mt.json", ""))
|
||||
# Find the main firmware .bin size (largest .bin, excluding OTA/littlefs/bleota)
|
||||
bin_size = None
|
||||
for entry in data.get("files", []):
|
||||
name = entry.get("name", "")
|
||||
if name.startswith("firmware-") and name.endswith(".bin"):
|
||||
bin_size = entry["bytes"]
|
||||
break
|
||||
# Fallback: any .bin that isn't ota/littlefs/bleota
|
||||
if bin_size is None:
|
||||
for entry in data.get("files", []):
|
||||
name = entry.get("name", "")
|
||||
if name.endswith(".bin") and not any(
|
||||
x in name for x in ["littlefs", "bleota", "ota"]
|
||||
):
|
||||
bin_size = entry["bytes"]
|
||||
break
|
||||
if bin_size is not None:
|
||||
sizes[board] = bin_size
|
||||
return sizes
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 3:
|
||||
print(f"Usage: {sys.argv[0]} <manifest_dir> <output.json>", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
manifest_dir = sys.argv[1]
|
||||
output_path = sys.argv[2]
|
||||
|
||||
sizes = collect_sizes(manifest_dir)
|
||||
with open(output_path, "w") as f:
|
||||
json.dump(sizes, f, indent=2, sort_keys=True)
|
||||
|
||||
print(f"Collected sizes for {len(sizes)} targets -> {output_path}")
|
||||
@@ -1,30 +0,0 @@
|
||||
---
|
||||
Lora:
|
||||
## Ebyte E80-900M22S
|
||||
## This is a bit experimental
|
||||
##
|
||||
##
|
||||
Module: lr1121
|
||||
gpiochip: 1 # subtract 32 from the gpio numbers
|
||||
DIO3_TCXO_VOLTAGE: 1.8
|
||||
CS: 16 #pin6 / GPIO48 1C0
|
||||
IRQ: 23 #pin17 / GPIO55 1C7
|
||||
Busy: 22 #pin16 / GPIO54 1C6
|
||||
Reset: 25 #pin13 / GPIO57 1D1
|
||||
|
||||
|
||||
spidev: spidev0.0 #pins are (CS=16, CLK=17, MOSI=18, MISO=19)
|
||||
spiSpeed: 2000000
|
||||
|
||||
rfswitch_table:
|
||||
pins: [DIO5, DIO6, DIO7]
|
||||
MODE_STBY: [LOW, LOW, LOW]
|
||||
MODE_RX: [LOW, HIGH, LOW]
|
||||
MODE_TX: [HIGH, HIGH, LOW]
|
||||
MODE_TX_HP: [HIGH, LOW, LOW]
|
||||
MODE_TX_HF: [LOW, LOW, LOW]
|
||||
MODE_GNSS: [LOW, LOW, HIGH]
|
||||
MODE_WIFI: [LOW, LOW, LOW]
|
||||
|
||||
General:
|
||||
MACAddressSource: eth0
|
||||
@@ -1,46 +0,0 @@
|
||||
---
|
||||
Lora:
|
||||
## Ebyte E80-900M22S
|
||||
## This is a bit experimental
|
||||
##
|
||||
##
|
||||
Module: lr1121
|
||||
gpiochip: 1 # subtract 32 from the gpio numbers
|
||||
DIO3_TCXO_VOLTAGE: 1.8
|
||||
CS: 16 #pin6 / GPIO48 1C0
|
||||
IRQ: 23 #pin17 / GPIO55 1C7
|
||||
Busy: 22 #pin16 / GPIO54 1C6
|
||||
Reset: 25 #pin13 / GPIO57 1D1
|
||||
|
||||
|
||||
spidev: spidev0.0 #pins are (CS=16, CLK=17, MOSI=18, MISO=19)
|
||||
spiSpeed: 2000000
|
||||
|
||||
rfswitch_table:
|
||||
pins:
|
||||
- DIO5
|
||||
- DIO6
|
||||
MODE_STBY:
|
||||
- LOW
|
||||
- LOW
|
||||
MODE_RX:
|
||||
- HIGH
|
||||
- LOW
|
||||
MODE_TX:
|
||||
- HIGH
|
||||
- HIGH
|
||||
MODE_TX_HP:
|
||||
- LOW
|
||||
- HIGH
|
||||
MODE_TX_HF:
|
||||
- LOW
|
||||
- LOW
|
||||
MODE_GNSS:
|
||||
- LOW
|
||||
- LOW
|
||||
MODE_WIFI:
|
||||
- LOW
|
||||
- LOW
|
||||
|
||||
General:
|
||||
MACAddressSource: eth0
|
||||
@@ -1,30 +0,0 @@
|
||||
---
|
||||
Lora:
|
||||
## Ebyte E80-900M22S
|
||||
## This is a bit experimental
|
||||
##
|
||||
##
|
||||
Module: lr1121
|
||||
gpiochip: 1 # subtract 32 from the gpio numbers
|
||||
DIO3_TCXO_VOLTAGE: 1.8
|
||||
CS: 16 #pin6 / GPIO48 1C0
|
||||
IRQ: 23 #pin17 / GPIO55 1C7
|
||||
Busy: 22 #pin16 / GPIO54 1C6
|
||||
Reset: 25 #pin13 / GPIO57 1D1
|
||||
|
||||
|
||||
spidev: spidev0.0 #pins are (CS=16, CLK=17, MOSI=18, MISO=19)
|
||||
spiSpeed: 2000000
|
||||
|
||||
rfswitch_table:
|
||||
pins: [DIO5, DIO6, DIO7]
|
||||
MODE_STBY: [LOW, LOW, LOW]
|
||||
MODE_RX: [LOW, LOW, LOW]
|
||||
MODE_TX: [LOW, HIGH, LOW]
|
||||
MODE_TX_HP: [HIGH, LOW, LOW]
|
||||
# MODE_TX_HF: []
|
||||
# MODE_GNSS: []
|
||||
MODE_WIFI: [LOW, LOW, LOW]
|
||||
|
||||
General:
|
||||
MACAddressSource: eth0
|
||||
@@ -5,9 +5,7 @@ Meta:
|
||||
- raspberry-pi
|
||||
|
||||
Lora:
|
||||
|
||||
### RAK13300 in Slot 2
|
||||
Module: sx1262
|
||||
### RAK13300 in Slot 2 pins
|
||||
IRQ: 18 #IO6
|
||||
Reset: 24 # IO4
|
||||
Busy: 19 # IO5
|
||||
@@ -15,7 +13,5 @@ Lora:
|
||||
Enable_Pins:
|
||||
- 26
|
||||
- 23
|
||||
DIO3_TCXO_VOLTAGE: true
|
||||
DIO2_AS_RF_SWITCH: true
|
||||
spidev: spidev0.1
|
||||
# CS: 7
|
||||
@@ -5,18 +5,14 @@ Meta:
|
||||
- raspberry-pi
|
||||
|
||||
Lora:
|
||||
|
||||
### RAK13302 in Slot 2
|
||||
Module: sx1262
|
||||
### RAK13302 in Slot 2 pins
|
||||
IRQ: 18 #IO6
|
||||
Reset: 24 # IO4
|
||||
Busy: 19 # IO5
|
||||
# Ant_sw: 23 # IO3
|
||||
# Ant_sw: 23 # IO3
|
||||
Enable_Pins:
|
||||
- 26
|
||||
- 23
|
||||
DIO3_TCXO_VOLTAGE: true
|
||||
DIO2_AS_RF_SWITCH: true
|
||||
spidev: spidev0.1
|
||||
# CS: 7
|
||||
TX_GAIN_LORA: [9, 9, 10, 11, 9, 8, 9, 10, 10, 10, 11, 12, 12, 12, 12, 12, 12, 12, 12, 10, 9, 8]
|
||||
@@ -1,18 +0,0 @@
|
||||
# Station G3 motherboard with a Raspberry Pi Zero 2W as the MCU daughterboard.
|
||||
# Verify spidev / I2C device paths for your OS — they may differ.
|
||||
Meta:
|
||||
name: Station G3
|
||||
support: community
|
||||
compatible:
|
||||
- raspberry-pi
|
||||
|
||||
Lora:
|
||||
Module: sx1262
|
||||
IRQ: 22 # BCM pin — wiki spec
|
||||
Reset: 16 # BCM pin — wiki spec
|
||||
Busy: 24 # BCM pin — wiki spec
|
||||
# CS: 8 # BCM 8 = SPI0 CE0 (default); uncomment only to override
|
||||
DIO2_AS_RF_SWITCH: true
|
||||
DIO3_TCXO_VOLTAGE: true
|
||||
spidev: spidev0.0
|
||||
# SX126X_MAX_POWER: 19 # matches Station G2 firmware cap; raise carefully per PA jumper mode
|
||||
@@ -1,439 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deterministic seed-data generator for the fake NodeDB fixture pipeline.
|
||||
|
||||
Writes a JSONL file describing N fake-but-realistic Meshtastic peers.
|
||||
The output is hand-editable and committed; a sibling compile step
|
||||
(bin/seed-json-to-proto.py) turns it into a binary `meshtastic_NodeDatabase`
|
||||
v25 protobuf with fresh "now-relative" timestamps.
|
||||
|
||||
Determinism contract:
|
||||
Same --seed -> byte-identical JSONL output, regardless of wall clock.
|
||||
All timestamps are stored as `*_offset_sec` (seconds before "now"); the
|
||||
compile step resolves them to absolute epochs at compile time.
|
||||
|
||||
Structural fields covered:
|
||||
* NodeInfoLite header: num, long_name, short_name, hw_model, role,
|
||||
public_key, snr, channel, hops_away, next_hop, bitfield flags
|
||||
* PositionLite: lat/long Gaussian around --centroid, altitude, source
|
||||
* DeviceMetrics: battery/voltage/util/uptime
|
||||
* EnvironmentMetrics: temp/humidity/pressure/iaq
|
||||
* StatusMessage: error_code (usually zero)
|
||||
|
||||
Active-board allow-list:
|
||||
hw_model values are restricted to the intersection of
|
||||
(a) variants with `custom_meshtastic_support_level = 1` in
|
||||
variants/*/*/platformio.ini, AND
|
||||
(b) values present in the `HardwareModel` enum in mesh.proto.
|
||||
See HW_MODEL_WEIGHTS below. Deprecated boards (legacy TLORA / Heltec V1-2 /
|
||||
classic TBEAM / TBEAM_V0P7 / Nano G1 / etc.) and fuzzer-only sentinels
|
||||
(PORTDUINO, ANDROID_SIM, DIY_V1, ...) are excluded.
|
||||
|
||||
Active-role allow-list:
|
||||
Excludes ROUTER_CLIENT (deprecated v2.3.15) and REPEATER (deprecated v2.7.11).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as _dt
|
||||
import json
|
||||
import math
|
||||
import pathlib
|
||||
import random
|
||||
import sys
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Active-board allow-list (intersection of tier-1 variants + HardwareModel enum).
|
||||
# Refresh by running:
|
||||
# for f in $(find variants -name 'platformio.ini' | xargs grep -lE 'custom_meshtastic_support_level = 1'); do
|
||||
# grep custom_meshtastic_hw_model_slug $f | awk -F= '{print $2}' | tr -d ' ';
|
||||
# done | sort -u | comm -12 - <(python3 -c "from meshtastic.protobuf.mesh_pb2 import HardwareModel; print('\\n'.join(HardwareModel.keys()))" | sort)
|
||||
# --------------------------------------------------------------------------
|
||||
HW_MODEL_WEIGHTS: dict[str, float] = {
|
||||
"HELTEC_V3": 14.0,
|
||||
"T_DECK": 9.0,
|
||||
"HELTEC_V4": 8.0,
|
||||
"RAK4631": 8.0,
|
||||
"HELTEC_MESH_POCKET": 6.0,
|
||||
"TRACKER_T1000_E": 5.0,
|
||||
"HELTEC_MESH_NODE_T114": 5.0,
|
||||
"T_DECK_PRO": 5.0,
|
||||
"LILYGO_TBEAM_S3_CORE": 4.0,
|
||||
"HELTEC_WIRELESS_PAPER": 4.0,
|
||||
"HELTEC_WSL_V3": 3.0,
|
||||
"T_ECHO": 3.0,
|
||||
"HELTEC_WIRELESS_TRACKER": 3.0,
|
||||
"HELTEC_WIRELESS_TRACKER_V2": 2.0,
|
||||
"HELTEC_VISION_MASTER_E290": 2.0,
|
||||
"HELTEC_MESH_SOLAR": 2.0,
|
||||
"SEEED_WIO_TRACKER_L1": 2.0,
|
||||
"T_LORA_PAGER": 1.5,
|
||||
"HELTEC_VISION_MASTER_E213": 1.5,
|
||||
"T_ECHO_PLUS": 1.0,
|
||||
"MUZI_BASE": 1.0,
|
||||
"WISMESH_TAP_V2": 1.0,
|
||||
"THINKNODE_M2": 1.0,
|
||||
"THINKNODE_M5": 1.0,
|
||||
"TLORA_T3_S3": 1.0,
|
||||
# Long tail (uniform low weight across remaining tier-1 boards):
|
||||
"HELTEC_V4_R8": 0.3,
|
||||
"HELTEC_VISION_MASTER_T190": 0.3,
|
||||
"HELTEC_HT62": 0.3,
|
||||
"HELTEC_MESH_NODE_T096": 0.3,
|
||||
"M5STACK_C6L": 0.3,
|
||||
"MINI_EPAPER_S3": 0.3,
|
||||
"MUZI_R1_NEO": 0.3,
|
||||
"NOMADSTAR_METEOR_PRO": 0.3,
|
||||
"RAK3312": 0.3,
|
||||
"RAK3401": 0.3,
|
||||
"SEEED_SOLAR_NODE": 0.3,
|
||||
"SEEED_WIO_TRACKER_L1_EINK": 0.3,
|
||||
"SENSECAP_INDICATOR": 0.3,
|
||||
"TBEAM_1_WATT": 0.3,
|
||||
"THINKNODE_M1": 0.3,
|
||||
"THINKNODE_M3": 0.3,
|
||||
"THINKNODE_M6": 0.3,
|
||||
"T_ECHO_LITE": 0.3,
|
||||
"WISMESH_TAG": 0.3,
|
||||
"WISMESH_TAP": 0.3,
|
||||
"XIAO_NRF52_KIT": 0.3,
|
||||
"CROWPANEL": 0.3,
|
||||
}
|
||||
|
||||
# Non-deprecated roles only.
|
||||
ROLE_WEIGHTS: dict[str, float] = {
|
||||
"CLIENT": 75.0,
|
||||
"CLIENT_MUTE": 5.0,
|
||||
"ROUTER": 7.0,
|
||||
"TRACKER": 3.0,
|
||||
"SENSOR": 2.0,
|
||||
"CLIENT_HIDDEN": 2.0,
|
||||
"ROUTER_LATE": 2.0,
|
||||
"CLIENT_BASE": 2.0,
|
||||
"TAK": 1.0,
|
||||
"TAK_TRACKER": 0.5,
|
||||
"LOST_AND_FOUND": 0.5,
|
||||
}
|
||||
|
||||
# Name pools — 60 firsts × 60 lasts = 3600 combinations.
|
||||
FIRSTS = [
|
||||
"Quick", "Brave", "Silent", "Wild", "Lone", "Bright", "Red", "Blue",
|
||||
"Green", "Black", "White", "Iron", "Steel", "Copper", "Silver", "Gold",
|
||||
"Stone", "River", "Forest", "Mountain", "Canyon", "Desert", "Storm", "Sky",
|
||||
"Solar", "Lunar", "Dawn", "Dusk", "Misty", "Frosty", "Sunny", "Shady",
|
||||
"Happy", "Sleepy", "Drowsy", "Sneaky", "Sharp", "Smooth", "Rough", "Loud",
|
||||
"Soft", "Slow", "Fast", "Tall", "Short", "Old", "New", "Tiny",
|
||||
"Giant", "Hidden", "Lost", "Found", "Wandering", "Roving", "Drifting", "Floating",
|
||||
"Burning", "Frozen", "Whispering", "Howling",
|
||||
]
|
||||
LASTS = [
|
||||
"Phoenix", "Lion", "Bear", "Wolf", "Hawk", "Eagle", "Fox", "Lynx",
|
||||
"Cougar", "Coyote", "Raven", "Owl", "Crow", "Falcon", "Heron", "Crane",
|
||||
"Otter", "Badger", "Bison", "Elk", "Moose", "Stag", "Doe", "Hare",
|
||||
"Marmot", "Mole", "Beaver", "Squirrel", "Mustang", "Bronco", "Pony", "Colt",
|
||||
"Cobra", "Viper", "Mamba", "Adder", "Gecko", "Iguana", "Tortoise", "Turtle",
|
||||
"Salmon", "Trout", "Bass", "Pike", "Shark", "Whale", "Dolphin", "Seal",
|
||||
"Cactus", "Yucca", "Sage", "Juniper", "Pine", "Cedar", "Aspen", "Oak",
|
||||
"Bluff", "Mesa", "Arroyo", "Ridge",
|
||||
]
|
||||
|
||||
# Brief callsign pool for licensed-looking suffixes.
|
||||
CALLSIGN_PREFIXES = ["KX", "WD", "N5", "KE", "AB", "W5", "K1", "KQ", "AE", "NM"]
|
||||
|
||||
# Only emojis that fit in 4 UTF-8 bytes (no variation selectors). short_name's
|
||||
# nanopb max_size:5 (incl. NUL) limits content to 4 bytes. ❄️ / ☀️ would be
|
||||
# 6 bytes due to U+FE0F variation selector — explicitly excluded.
|
||||
EMOJI_SHORTNAMES = ["🦊", "🐺", "🦅", "🐢", "🌵", "🔥", "🌙",
|
||||
"🌊", "🗻", "🌲", "🦌", "🐝", "🦂", "🦉",
|
||||
"🦇", "🦋"]
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
NUM_RESERVED = 4 # firmware reserves 0..3 (per NodeDB constants)
|
||||
NUM_MAX_EXCLUSIVE = 0x80000000 # restrict to positive int32 range for readability
|
||||
|
||||
|
||||
def _weighted_choice(rng: random.Random, weights: dict[str, float]) -> str:
|
||||
"""Deterministic weighted pick. Uses sorted keys so dict order is fixed."""
|
||||
keys = sorted(weights.keys())
|
||||
totals = [weights[k] for k in keys]
|
||||
return rng.choices(keys, weights=totals, k=1)[0]
|
||||
|
||||
|
||||
def _gen_long_name(rng: random.Random, is_licensed: bool) -> str:
|
||||
base = f"{rng.choice(FIRSTS)} {rng.choice(LASTS)}"
|
||||
if is_licensed:
|
||||
prefix = rng.choice(CALLSIGN_PREFIXES)
|
||||
# Two trailing alpha chars after the digit; keep within 25 - len(base) - 1
|
||||
suffix = f" {prefix}{rng.randint(0,9)}{rng.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ')}{rng.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ')}"
|
||||
# nanopb max_size:25 means C string fits 24 bytes + NUL.
|
||||
if len(base) + len(suffix) <= 24:
|
||||
base = base + suffix
|
||||
# Hard cap to 24 chars (nanopb max_size:25 minus NUL).
|
||||
return base[:24]
|
||||
|
||||
|
||||
def _gen_short_name(rng: random.Random, long_name: str) -> str:
|
||||
# 10% emoji-only short_name
|
||||
if rng.random() < 0.10:
|
||||
return rng.choice(EMOJI_SHORTNAMES)
|
||||
first_char = long_name[0].upper() if long_name else "X"
|
||||
alphanums = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
||||
return first_char + "".join(rng.choices(alphanums, k=3))
|
||||
|
||||
|
||||
def _gen_hops_away(rng: random.Random) -> int:
|
||||
# Geometric-ish: 0→55%, 1→25%, 2→12%, 3→5%, 4→2%, 5+→1%
|
||||
r = rng.random()
|
||||
if r < 0.55:
|
||||
return 0
|
||||
if r < 0.80:
|
||||
return 1
|
||||
if r < 0.92:
|
||||
return 2
|
||||
if r < 0.97:
|
||||
return 3
|
||||
if r < 0.99:
|
||||
return 4
|
||||
return rng.randint(5, 7)
|
||||
|
||||
|
||||
def _gen_position(
|
||||
rng: random.Random,
|
||||
centroid_lat: float,
|
||||
centroid_lon: float,
|
||||
spread_km: float,
|
||||
last_heard_offset_sec: int,
|
||||
) -> dict:
|
||||
# 1 deg ≈ 111 km at the equator; we use this as a flat approximation.
|
||||
lat = centroid_lat + rng.gauss(0.0, spread_km / 111.0)
|
||||
lon = centroid_lon + rng.gauss(0.0, spread_km / 111.0)
|
||||
altitude = max(0, round(rng.gauss(1376.0, 250.0))) # T or C valley floor + relief
|
||||
# Position was reported up to 300s before last_heard.
|
||||
time_offset_sec = last_heard_offset_sec + rng.randint(0, 300)
|
||||
return {
|
||||
"latitude": round(lat, 6),
|
||||
"longitude": round(lon, 6),
|
||||
"altitude": altitude,
|
||||
"time_offset_sec": time_offset_sec,
|
||||
"location_source": "LOC_INTERNAL",
|
||||
}
|
||||
|
||||
|
||||
def _gen_telemetry(rng: random.Random) -> dict:
|
||||
# 5% plugged-in (battery_level == 101); rest uniform [10..100].
|
||||
if rng.random() < 0.05:
|
||||
battery_level = 101
|
||||
voltage = 4.20
|
||||
else:
|
||||
battery_level = rng.randint(10, 100)
|
||||
voltage = round(3.3 + (battery_level / 100.0) * 0.9, 3)
|
||||
# Beta distributions for low/right-skewed metrics; randomly draw via gammavariate.
|
||||
def _beta(a: float, b: float) -> float:
|
||||
x = rng.gammavariate(a, 1.0)
|
||||
y = rng.gammavariate(b, 1.0)
|
||||
return x / (x + y)
|
||||
channel_utilization = round(_beta(2.0, 15.0) * 100.0, 2)
|
||||
air_util_tx = round(_beta(1.5, 20.0) * 10.0, 3)
|
||||
uptime_seconds = int(rng.expovariate(1.0 / 86400.0))
|
||||
return {
|
||||
"battery_level": battery_level,
|
||||
"voltage": voltage,
|
||||
"channel_utilization": channel_utilization,
|
||||
"air_util_tx": air_util_tx,
|
||||
"uptime_seconds": uptime_seconds,
|
||||
}
|
||||
|
||||
|
||||
def _gen_environment(rng: random.Random) -> dict:
|
||||
return {
|
||||
"temperature": round(rng.gauss(22.0, 8.0), 2),
|
||||
"relative_humidity": round(min(100.0, max(0.0, rng.gauss(55.0, 20.0))), 2),
|
||||
"barometric_pressure": round(rng.gauss(1013.0, 8.0), 2),
|
||||
"iaq": int(min(500, max(0, round(rng.gauss(50.0, 30.0))))),
|
||||
}
|
||||
|
||||
|
||||
def _gen_status(rng: random.Random) -> dict:
|
||||
# `StatusMessage` (mesh.proto:1445) has a single free-form `string status`.
|
||||
# Most peers report a healthy short status; occasional alert string.
|
||||
healthy = ["OK", "online", "active", "running", "ready", "nominal"]
|
||||
alert = ["low-batt", "no-gps", "weak-signal", "rebooted", "offline-soon"]
|
||||
if rng.random() < 0.92:
|
||||
return {"status": rng.choice(healthy)}
|
||||
return {"status": rng.choice(alert)}
|
||||
|
||||
|
||||
def _gen_node(
|
||||
rng: random.Random,
|
||||
num: int,
|
||||
centroid_lat: float,
|
||||
centroid_lon: float,
|
||||
spread_km: float,
|
||||
coverage: dict[str, float],
|
||||
last_heard_mean_sec: int,
|
||||
last_heard_max_sec: int,
|
||||
) -> dict:
|
||||
is_licensed = rng.random() < 0.05
|
||||
long_name = _gen_long_name(rng, is_licensed)
|
||||
short_name = _gen_short_name(rng, long_name)
|
||||
hw_model = _weighted_choice(rng, HW_MODEL_WEIGHTS)
|
||||
role = _weighted_choice(rng, ROLE_WEIGHTS)
|
||||
has_public_key = rng.random() < 0.92
|
||||
public_key_hex = (
|
||||
"".join(f"{rng.randint(0,255):02x}" for _ in range(32)) if has_public_key else ""
|
||||
)
|
||||
snr = round(max(-20.0, min(12.0, rng.gauss(6.0, 4.0))), 2)
|
||||
channel = 0 if rng.random() < 0.90 else rng.randint(1, 7)
|
||||
hops_away = _gen_hops_away(rng)
|
||||
next_hop = rng.randint(0, 255) if hops_away > 0 else 0
|
||||
last_heard_offset_sec = int(min(rng.expovariate(1.0 / last_heard_mean_sec), last_heard_max_sec))
|
||||
|
||||
bitfield = {
|
||||
"has_user": True,
|
||||
"is_favorite": rng.random() < 0.08,
|
||||
"is_muted": rng.random() < 0.03,
|
||||
"via_mqtt": rng.random() < 0.12,
|
||||
"is_ignored": rng.random() < 0.01,
|
||||
"is_licensed": is_licensed,
|
||||
"has_is_unmessagable": True,
|
||||
"is_unmessagable": rng.random() < 0.02,
|
||||
"is_key_manually_verified": rng.random() < 0.04,
|
||||
}
|
||||
|
||||
node: dict = {
|
||||
"num": f"0x{num:08x}",
|
||||
"long_name": long_name,
|
||||
"short_name": short_name,
|
||||
"hw_model": hw_model,
|
||||
"role": role,
|
||||
"public_key_hex": public_key_hex,
|
||||
"snr": snr,
|
||||
"channel": channel,
|
||||
"hops_away": hops_away,
|
||||
"next_hop": next_hop,
|
||||
"last_heard_offset_sec": last_heard_offset_sec,
|
||||
"bitfield": bitfield,
|
||||
"position": (
|
||||
_gen_position(rng, centroid_lat, centroid_lon, spread_km, last_heard_offset_sec)
|
||||
if rng.random() < coverage["position"]
|
||||
else None
|
||||
),
|
||||
"telemetry": _gen_telemetry(rng) if rng.random() < coverage["telemetry"] else None,
|
||||
"environment": _gen_environment(rng) if rng.random() < coverage["environment"] else None,
|
||||
"status": _gen_status(rng) if rng.random() < coverage["status"] else None,
|
||||
}
|
||||
return node
|
||||
|
||||
|
||||
def _parse_my_node_num(s: str | None) -> int | None:
|
||||
if s is None:
|
||||
return None
|
||||
s = s.strip()
|
||||
if s.startswith("0x") or s.startswith("0X"):
|
||||
return int(s, 16)
|
||||
return int(s)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
p = argparse.ArgumentParser(
|
||||
description="Deterministic JSONL seed for the fake NodeDB fixture.",
|
||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
||||
)
|
||||
p.add_argument("--count", type=int, required=True, help="Number of fake nodes to emit.")
|
||||
p.add_argument("--seed", type=int, required=True, help="Deterministic seed.")
|
||||
p.add_argument("--out", required=True, help="Output JSONL path.")
|
||||
p.add_argument(
|
||||
"--centroid",
|
||||
default="33.1284,-107.2528",
|
||||
help="LAT,LON centroid (default: Truth or Consequences, NM).",
|
||||
)
|
||||
p.add_argument("--spread-km", type=float, default=60.0, help="Gaussian std-dev in km.")
|
||||
p.add_argument("--position-coverage", type=float, default=0.85)
|
||||
p.add_argument("--telemetry-coverage", type=float, default=0.70)
|
||||
p.add_argument("--environment-coverage", type=float, default=0.25)
|
||||
p.add_argument("--status-coverage", type=float, default=0.40)
|
||||
p.add_argument("--my-node-num", default=None, help="Exclude this NodeNum from generated set (hex or dec).")
|
||||
p.add_argument("--last-heard-mean-sec", type=int, default=3600)
|
||||
p.add_argument("--last-heard-max-sec", type=int, default=7 * 86400)
|
||||
args = p.parse_args(argv)
|
||||
|
||||
if args.count <= 0:
|
||||
print("--count must be positive", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
try:
|
||||
centroid_lat, centroid_lon = (float(s) for s in args.centroid.split(","))
|
||||
except ValueError:
|
||||
print(f"--centroid must be LAT,LON; got {args.centroid!r}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
my_node_num = _parse_my_node_num(args.my_node_num)
|
||||
|
||||
rng = random.Random(args.seed)
|
||||
|
||||
# 1) Generate a unique deterministic set of NodeNums.
|
||||
nums: set[int] = set()
|
||||
while len(nums) < args.count:
|
||||
n = rng.randrange(NUM_RESERVED, NUM_MAX_EXCLUSIVE)
|
||||
if my_node_num is not None and n == my_node_num:
|
||||
continue
|
||||
nums.add(n)
|
||||
ordered_nums = sorted(nums) # sort to fix output order independent of set hash
|
||||
|
||||
# 2) Per-node generation (in num order, single RNG continues).
|
||||
coverage = {
|
||||
"position": args.position_coverage,
|
||||
"telemetry": args.telemetry_coverage,
|
||||
"environment": args.environment_coverage,
|
||||
"status": args.status_coverage,
|
||||
}
|
||||
nodes = [
|
||||
_gen_node(
|
||||
rng,
|
||||
n,
|
||||
centroid_lat,
|
||||
centroid_lon,
|
||||
args.spread_km,
|
||||
coverage,
|
||||
args.last_heard_mean_sec,
|
||||
args.last_heard_max_sec,
|
||||
)
|
||||
for n in ordered_nums
|
||||
]
|
||||
|
||||
# 3) Write JSONL.
|
||||
out_path = pathlib.Path(args.out)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
# `generated_at_iso` is informational; it does NOT affect determinism because
|
||||
# we derive it from the seed, not from wall clock. (Same seed -> same string.)
|
||||
generated_at = _dt.datetime.fromtimestamp(args.seed, tz=_dt.timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
meta = {
|
||||
"_meta": {
|
||||
"version": 25,
|
||||
"seed": args.seed,
|
||||
"count": args.count,
|
||||
"centroid": [centroid_lat, centroid_lon],
|
||||
"spread_km": args.spread_km,
|
||||
"generated_at_iso": generated_at,
|
||||
"my_node_num_excluded": (None if my_node_num is None else f"0x{my_node_num:08x}"),
|
||||
"coverage": coverage,
|
||||
"last_heard_mean_sec": args.last_heard_mean_sec,
|
||||
"last_heard_max_sec": args.last_heard_max_sec,
|
||||
}
|
||||
}
|
||||
with out_path.open("w", encoding="utf-8") as f:
|
||||
# `ensure_ascii=False` so emoji short_names survive. `sort_keys=True` for
|
||||
# determinism (insertion order varies by Python version otherwise).
|
||||
f.write(json.dumps(meta, ensure_ascii=False, sort_keys=True) + "\n")
|
||||
for node in nodes:
|
||||
f.write(json.dumps(node, ensure_ascii=False, sort_keys=True) + "\n")
|
||||
|
||||
print(f"wrote {args.count} nodes to {out_path} ({out_path.stat().st_size} bytes)", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -293,12 +293,9 @@ if ("HAS_TFT", 1) in env.get("CPPDEFINES", []):
|
||||
board_arch = infer_architecture(env.BoardConfig())
|
||||
should_skip_manifest = board_arch is None
|
||||
|
||||
# Most platforms can generate the manifest as part of the default 'buildprog' target.
|
||||
# Typically this passes success/failure properly.
|
||||
mtjson_deps = ["buildprog"]
|
||||
if platform.name == "espressif32":
|
||||
# On ESP32, we need to explicitly depend upon the binary to prevent fake-success upon failure.
|
||||
mtjson_deps = ["$BUILD_DIR/${PROGNAME}.bin"]
|
||||
# For host/native envs, avoid depending on 'buildprog' (some targets don't define it)
|
||||
mtjson_deps = [] if should_skip_manifest else ["buildprog"]
|
||||
if not should_skip_manifest and platform.name == "espressif32":
|
||||
# Build littlefs image as part of mtjson target
|
||||
# Equivalent to `pio run -t buildfs`
|
||||
target_lfs = env.DataToBin(
|
||||
@@ -312,8 +309,7 @@ if should_skip_manifest:
|
||||
|
||||
env.AddCustomTarget(
|
||||
name="mtjson",
|
||||
# For host/native envs, avoid depending on 'buildprog' (some targets don't define it)
|
||||
dependencies=[],
|
||||
dependencies=mtjson_deps,
|
||||
actions=[skip_manifest],
|
||||
title="Meshtastic Manifest (skipped)",
|
||||
description="mtjson generation is skipped for native environments",
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Regenerate the fake-NodeDB fixtures: produces 250 / 500 / 1000 / 2000-node
|
||||
# JSONL seed files + their compiled v25 protobufs.
|
||||
#
|
||||
# Layout:
|
||||
# test/fixtures/nodedb/seed_v25_<N>.jsonl — COMMITTED, hand-editable.
|
||||
# build/fixtures/nodedb/nodes_v25_<N>.proto — .gitignored, build artifact.
|
||||
# Drop into /prefs/nodes.proto.
|
||||
#
|
||||
# Daily use: ./bin/regen-fake-nodedbs.sh
|
||||
# - Recompiles protos from committed seeds (fresh wall-clock timestamps).
|
||||
# Intentional seed bump: REGEN_SEEDS=yes ./bin/regen-fake-nodedbs.sh
|
||||
# - Overwrites the committed JSONL files with freshly-seeded data.
|
||||
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# 1) Make sure the Python protobuf bindings exist (in-tree generation; .gitignored).
|
||||
if [[ ! -d bin/_generated/meshtastic ]]; then
|
||||
echo "regenerating Python protobuf bindings (one-time)..."
|
||||
./bin/regen-py-protos.sh
|
||||
fi
|
||||
|
||||
# 2) Pick a Python interpreter that has the meshtastic deps installed.
|
||||
# Prefer the mcp-server venv (most likely to be set up by the operator).
|
||||
PY="python3"
|
||||
for cand in mcp-server/.venv/bin/python3 .venv/bin/python3; do
|
||||
if [[ -x "$cand" ]]; then
|
||||
PY="$cand"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
# 3) Pinned seeds per size — bump only when you intentionally want different
|
||||
# structural data committed. Parallel arrays so the script works on
|
||||
# macOS bash 3.2 (no `declare -A`).
|
||||
SIZES=(250 500 1000 2000)
|
||||
SEEDS=(20260511 20260512 20260513 20260514)
|
||||
|
||||
REGEN_SEEDS="${REGEN_SEEDS:-no}"
|
||||
|
||||
mkdir -p build/fixtures/nodedb test/fixtures/nodedb
|
||||
|
||||
for i in 0 1 2 3; do
|
||||
n="${SIZES[$i]}"
|
||||
seed="${SEEDS[$i]}"
|
||||
jsonl=$(printf "test/fixtures/nodedb/seed_v25_%04d.jsonl" "$n")
|
||||
proto=$(printf "build/fixtures/nodedb/nodes_v25_%04d.proto" "$n")
|
||||
|
||||
if [[ "$REGEN_SEEDS" == "yes" || ! -f "$jsonl" ]]; then
|
||||
$PY bin/gen-fake-nodedb-seed.py \
|
||||
--count "$n" \
|
||||
--seed "$seed" \
|
||||
--out "$jsonl" \
|
||||
--centroid 33.1284,-107.2528 \
|
||||
--spread-km 60 \
|
||||
--position-coverage 0.85 \
|
||||
--telemetry-coverage 0.70 \
|
||||
--environment-coverage 0.25 \
|
||||
--status-coverage 0.40
|
||||
echo " seed: $jsonl ($(wc -c < "$jsonl") bytes)"
|
||||
fi
|
||||
|
||||
$PY bin/seed-json-to-proto.py --in "$jsonl" --out "$proto"
|
||||
echo " proto: $proto ($(wc -c < "$proto") bytes)"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Done. To load on Portduino native:"
|
||||
echo " cp build/fixtures/nodedb/nodes_v25_1000.proto ~/.portduino/default/prefs/nodes.proto"
|
||||
echo ""
|
||||
echo "To push to a hardware device:"
|
||||
echo " Use the mcp-server tool: push_fake_nodedb(size=1000, target=\"hardware\", port=\"/dev/cu.usbmodemXXXX\", confirm=True)"
|
||||
@@ -1,51 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Regenerate Python protobuf bindings from the in-tree `protobufs/` submodule
|
||||
# into `bin/_generated/`. Called by bin/regen-fake-nodedbs.sh; also useful as
|
||||
# a standalone refresh after any change to a .proto file.
|
||||
#
|
||||
# Output is .gitignored — bindings are a build artifact.
|
||||
#
|
||||
# Namespace rewrite:
|
||||
# The .proto files declare `package meshtastic;`, which makes protoc emit
|
||||
# imports like `from meshtastic import mesh_pb2`. That conflicts with the
|
||||
# PyPI `meshtastic` package (which the mcp-server relies on for its
|
||||
# SerialInterface/BLEInterface transport). We post-process the generated
|
||||
# files to live under `meshtastic_v25` instead — both the directory layout
|
||||
# and all internal imports — so they coexist cleanly with the PyPI package.
|
||||
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
if ! command -v protoc >/dev/null 2>&1; then
|
||||
echo "ERROR: protoc not found in PATH." >&2
|
||||
echo " macOS: brew install protobuf" >&2
|
||||
echo " Ubuntu/Debian: apt install protobuf-compiler" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
OUT=bin/_generated
|
||||
LOCAL_NS=meshtastic_v25
|
||||
|
||||
rm -rf "$OUT"
|
||||
mkdir -p "$OUT"
|
||||
|
||||
# 1) Generate from the in-tree protos. nanopb.proto first so its descriptor
|
||||
# is available for the [(nanopb).*] options on other messages.
|
||||
protoc \
|
||||
--proto_path=protobufs \
|
||||
--python_out="$OUT" \
|
||||
protobufs/nanopb.proto \
|
||||
protobufs/meshtastic/*.proto
|
||||
|
||||
# 2) Move the generated `meshtastic/` directory to `meshtastic_v25/`.
|
||||
mv "$OUT/meshtastic" "$OUT/$LOCAL_NS"
|
||||
|
||||
# 3) Rewrite internal imports: any reference to `meshtastic.X_pb2` or
|
||||
# `from meshtastic import X_pb2` becomes `meshtastic_v25.*`.
|
||||
python3 bin/_rewrite_proto_namespace.py "$OUT/$LOCAL_NS" "$LOCAL_NS"
|
||||
|
||||
# 4) Make the package importable.
|
||||
touch "$OUT/__init__.py"
|
||||
touch "$OUT/$LOCAL_NS/__init__.py"
|
||||
|
||||
echo "regenerated Python protobuf bindings -> $OUT/$LOCAL_NS/ (namespace: $LOCAL_NS)" >&2
|
||||
@@ -1,249 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run native PlatformIO unit tests and emit a single, unambiguous RED/AMBER/GREEN verdict.
|
||||
#
|
||||
# Why this exists: PlatformIO reports failures three different ways ([FAILED], :FAIL:,
|
||||
# [ERRORED]) and an all-pass run prints "N succeeded" with NO "0 failed" clause — so naive
|
||||
# greps produce false greens (see .notes/test-passfail-filter.md). This script encodes the
|
||||
# correct logic once, and cross-checks the number of suites that actually ran against the
|
||||
# canonical set in test/ so a suite silently going missing shows up as AMBER, not green.
|
||||
#
|
||||
# Usage:
|
||||
# ./bin/run-tests.sh # run all suites, full RAG + count cross-check
|
||||
# ./bin/run-tests.sh -f test_utf8 # run one suite (no count cross-check)
|
||||
# ./bin/run-tests.sh -e native # override env (default: coverage)
|
||||
# ./bin/run-tests.sh --quiet # only print the final RESULT line
|
||||
#
|
||||
# Exit codes: 0 = GREEN, 1 = RED, 2 = AMBER.
|
||||
#
|
||||
# The final line is machine-readable, e.g.:
|
||||
# RESULT: GREEN 19/19 suites passed
|
||||
# RESULT: AMBER 17/19 suites ran (missing: test_radio test_serial) — all that ran passed
|
||||
# RESULT: RED test_traffic_management: 1 failed (or: build/crash error)
|
||||
# RESULT: RED sanitizer fault — SUMMARY: AddressSanitizer: 1272 byte(s) leaked (tests may have
|
||||
# all passed; the coverage build aborts at exit on an ASan/LSan fault — often shown only
|
||||
# as [ERRORED]/SIGHUP. The script names it and points at running the binary bare.)
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
ENV="coverage"
|
||||
FILTER=""
|
||||
QUIET=false
|
||||
PASSTHRU=()
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
-f)
|
||||
FILTER="$2"
|
||||
PASSTHRU+=("-f" "$2")
|
||||
shift 2
|
||||
;;
|
||||
-e)
|
||||
ENV="$2"
|
||||
shift 2
|
||||
;;
|
||||
--quiet)
|
||||
QUIET=true
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
PASSTHRU+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Locate pio (PATH, then the standard PlatformIO venv).
|
||||
PIO="$(command -v pio || command -v platformio || echo "$HOME/.platformio/penv/bin/pio")"
|
||||
if [[ ! -x $PIO ]] && ! command -v "$PIO" >/dev/null 2>&1; then
|
||||
echo "RESULT: RED pio not found (looked in PATH and ~/.platformio/penv/bin)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
LOG="$(mktemp -t meshtest.XXXXXX.log)"
|
||||
MARKER=""
|
||||
PROGRESS_PID=""
|
||||
trap 'rm -f "$LOG" "${MARKER:-}"; [[ -n ${PROGRESS_PID:-} ]] && kill "$PROGRESS_PID" 2>/dev/null' EXIT
|
||||
|
||||
# Canonical suite set = the directories in test/. This is the source of truth for
|
||||
# "what should run"; a filtered run only expects its filtered suite.
|
||||
mapfile -t ALL_SUITES < <(find test -maxdepth 1 -type d -name 'test_*' -printf '%f\n' | sort)
|
||||
EXPECTED_COUNT=${#ALL_SUITES[@]}
|
||||
|
||||
# Cached object-count for this env, written after each completed build (in the gitignored build
|
||||
# dir). Used as the progress denominator: accurate for a full rebuild (every object recompiles),
|
||||
# only a rough upper bound for an incremental run.
|
||||
BASELINE_FILE=".pio/build/${ENV}/.runtests-objcount"
|
||||
|
||||
# Progress trail file (gitignored build dir). ALWAYS written so a backgrounded/piped run can be
|
||||
# checked mid-build with `tail -f` — that's the whole point: don't fly blind on a 20-min rebuild.
|
||||
PROGRESS_FILE=".pio/build/${ENV}/.runtests-progress"
|
||||
|
||||
# --- Progress heartbeat ------------------------------------------------------
|
||||
# Emit ONE status line every few seconds: build = objects (re)compiled this run / cached total +
|
||||
# best-effort ETA; test = suites finished / expected. Appends to $PROGRESS_FILE always (tail it to
|
||||
# check on a backgrounded run); also live-updates the tty when $5=1 (interactive --quiet). Never
|
||||
# touches $LOG, which is parsed for the verdict, so piped/CI captures stay clean.
|
||||
progress_monitor() {
|
||||
local marker="$1" objtotal="$2" testtotal="$3" pfile="$4" totty="$5" start now el done ran eta line
|
||||
start=$(date +%s)
|
||||
while :; do
|
||||
now=$(date +%s)
|
||||
el=$((now - start))
|
||||
if grep -q 'Testing\.\.\.' "$LOG" 2>/dev/null; then
|
||||
ran=$(grep -cE "${ENV}:test_[a-z0-9_]+ \[(PASSED|FAILED|ERRORED)\]" "$LOG" 2>/dev/null)
|
||||
line=$(printf '[test] %s/%s suites done — %dm%02ds' "$ran" "$testtotal" $((el / 60)) $((el % 60)))
|
||||
else
|
||||
done=$(find ".pio/build/${ENV}" -name '*.o' -newer "$marker" 2>/dev/null | wc -l)
|
||||
if ((objtotal > 0 && done > 0)); then
|
||||
eta=$((objtotal > done ? (objtotal - done) * el / done : 0))
|
||||
line=$(printf '[build] %d/%d objs — %dm%02ds — ETA ~%dm%02ds' \
|
||||
"$done" "$objtotal" $((el / 60)) $((el % 60)) $((eta / 60)) $((eta % 60)))
|
||||
else
|
||||
# done==0 (incremental: nothing to rebuild yet) or no cached baseline — no ETA yet.
|
||||
line=$(printf '[build] %d objs compiled — %dm%02ds' "$done" $((el / 60)) $((el % 60)))
|
||||
fi
|
||||
fi
|
||||
printf '%s\n' "$line" >>"$pfile" 2>/dev/null # file trail (always)
|
||||
[[ $totty == 1 ]] && printf '\r\033[K%s' "$line" >/dev/tty 2>/dev/null # live line (human)
|
||||
sleep 4
|
||||
done
|
||||
}
|
||||
|
||||
# Launch the heartbeat for every run. It writes the progress file unconditionally; the live tty
|
||||
# line only when interactive AND --quiet (where pio's own output is hidden — otherwise pio's
|
||||
# streamed compile lines already show progress and a \r line would just fight them).
|
||||
mkdir -p ".pio/build/${ENV}" 2>/dev/null || true
|
||||
: >"$PROGRESS_FILE" 2>/dev/null || true
|
||||
MARKER="$(mktemp -t meshtest-mark.XXXXXX)"
|
||||
TOTTY=0
|
||||
{ $QUIET && [[ -t 1 ]]; } && TOTTY=1
|
||||
progress_monitor "$MARKER" "$(cat "$BASELINE_FILE" 2>/dev/null || echo 0)" \
|
||||
"$([[ -n $FILTER ]] && echo 1 || echo "$EXPECTED_COUNT")" "$PROGRESS_FILE" "$TOTTY" &
|
||||
PROGRESS_PID=$!
|
||||
|
||||
if ! $QUIET; then
|
||||
echo "Running: $PIO test -e $ENV ${PASSTHRU[*]-} (expecting $EXPECTED_COUNT suites)"
|
||||
fi
|
||||
echo "progress: tail -f $PROGRESS_FILE" >&2
|
||||
|
||||
# Run pio, tee to log. PIPESTATUS[0] is pio's real exit (NOT tee's).
|
||||
if $QUIET; then
|
||||
"$PIO" test -e "$ENV" "${PASSTHRU[@]}" >"$LOG" 2>&1
|
||||
else
|
||||
"$PIO" test -e "$ENV" "${PASSTHRU[@]}" 2>&1 | tee "$LOG"
|
||||
fi
|
||||
PIO_RC=${PIPESTATUS[0]}
|
||||
|
||||
# Stop the heartbeat, clear its line, and cache this build's object total for next time.
|
||||
if [[ -n $PROGRESS_PID ]]; then
|
||||
kill "$PROGRESS_PID" 2>/dev/null
|
||||
wait "$PROGRESS_PID" 2>/dev/null
|
||||
PROGRESS_PID=""
|
||||
# Clear the live line only if we were writing one — opening /dev/tty when there is none is
|
||||
# itself a redirect-open error the trailing 2>/dev/null cannot suppress.
|
||||
[[ $TOTTY == 1 ]] && printf '\r\033[K' >/dev/tty 2>/dev/null
|
||||
fi
|
||||
[[ -d ".pio/build/${ENV}" ]] && find ".pio/build/${ENV}" -name '*.o' 2>/dev/null | wc -l >"$BASELINE_FILE" 2>/dev/null || true
|
||||
|
||||
# --- Outcome detection -------------------------------------------------------
|
||||
# The SAME outcome is spelled differently depending on which layer emitted the line — this is
|
||||
# the trap that produces false greens (grepping ":PASS" misses pio's "[PASSED]", grepping
|
||||
# "[FAILED]" misses Unity's ":FAIL:"). So every regex below matches BOTH spellings:
|
||||
# pass: Unity per-assertion ":PASS" | pio per-suite "[PASSED]" | summary "N succeeded"
|
||||
# fail: Unity per-assertion ":FAIL:" | pio per-suite "[FAILED]" | summary "M failed"
|
||||
# error: pio build/crash "[ERRORED]" | Unity "M Failures" | compiler "error:"
|
||||
# Match \b after :PASS/:FAIL so ":PASSED"/":FAILED" forms are also caught either way.
|
||||
FAIL_RE=':FAIL\b|\[FAILED\]|\[ERRORED\]|[1-9][0-9]* failed|[0-9]+ Tests [1-9][0-9]* Failures|error:|undefined reference|Segmentation fault|terminate called|SIGHUP|SIGSEGV|SIGABRT'
|
||||
# Positive proof tests actually ran & passed (absence != success). Accept any pass spelling:
|
||||
# the per-test/per-suite tokens OR a success summary line.
|
||||
PASS_RE=':PASS\b|\[PASSED\]|test cases: *[0-9]+ succeeded|[0-9]+ Tests 0 Failures'
|
||||
# Sanitizer (ASan/LSan/UBSan/TSan) fault signatures. The coverage build is sanitizer-instrumented
|
||||
# and aborts NON-ZERO at exit on a fault — most often a LeakSanitizer leak — AFTER every test has
|
||||
# already printed [PASSED]. pio then reports [ERRORED]/SIGHUP with no :FAIL: anywhere, so it
|
||||
# masquerades as a phantom "N-1 of N succeeded". See .notes/test-passfail-filter.md.
|
||||
# Match only real FAULT lines, never the benign "AddressSanitizer: failed to intercept '...'"
|
||||
# startup noise that prints on every sanitizer run (it'd mislabel a normal [FAILED] as a leak).
|
||||
# Formats per LLVM/Google sanitizer docs: ASan/LSan emit "==PID==ERROR: <San>: ...", UBSan emits
|
||||
# "file:line:col: runtime error: ...", TSan emits "WARNING: ThreadSanitizer: ..."; all close with
|
||||
# a "SUMMARY: <San>: ..." line (LSan-under-ASan reports its SUMMARY as "AddressSanitizer").
|
||||
SAN_RE='(ERROR|WARNING): (Address|Leak|Thread|UndefinedBehavior)Sanitizer:|SUMMARY: (Address|Leak|Thread|UndefinedBehavior)Sanitizer:|Direct leak of|Indirect leak of|detected memory leaks|heap-use-after-free|heap-buffer-overflow|stack-buffer-overflow|attempting double-free|LeakSanitizer has encountered a fatal error|runtime error:'
|
||||
|
||||
# Suites that produced a per-suite verdict. pio emits "coverage:test_x [PASSED|FAILED|ERRORED]";
|
||||
# a SKIPPED suite (hardware-only on native) is "accounted for" too, so it doesn't read as missing.
|
||||
mapfile -t RAN_SUITES < <(grep -oE "${ENV}:test_[a-z0-9_]+ \[(PASSED|FAILED|ERRORED)\]" "$LOG" |
|
||||
sed -E "s/^${ENV}:(test_[a-z0-9_]+) .*/\1/" | sort -u)
|
||||
RAN_COUNT=${#RAN_SUITES[@]}
|
||||
# Suites pio explicitly skipped (don't count these as "missing" in the canonical cross-check).
|
||||
mapfile -t SKIPPED_SUITES < <(grep -oE "${ENV}:test_[a-z0-9_]+.*\bSKIPPED\b" "$LOG" |
|
||||
grep -oE "test_[a-z0-9_]+" | sort -u)
|
||||
|
||||
verdict_red() {
|
||||
local detail bin
|
||||
detail="$(grep -nE '\[FAILED\]|:FAIL:|\[ERRORED\]' "$LOG" | head -3 | sed 's/^/ /')"
|
||||
echo ""
|
||||
echo "RED — failures detected:"
|
||||
[[ -n $detail ]] && echo "$detail"
|
||||
grep -E 'test cases:' "$LOG" | tail -1 | sed 's/^/ /'
|
||||
|
||||
# Path to the test binary for the "run it bare" hint. For native/coverage the test program is
|
||||
# the env executable (e.g. .pio/build/coverage/meshtasticd), NOT a file named 'program'.
|
||||
bin="$(find ".pio/build/${ENV}" -maxdepth 1 -type f -executable ! -name '*.so' 2>/dev/null | head -1)"
|
||||
[[ -z $bin ]] && bin=".pio/build/${ENV}/<program> (build it first: $PIO test -e ${ENV} ${FILTER:+-f $FILTER} --without-testing)"
|
||||
|
||||
# Sanitizer fault (ASan/LSan/UBSan/TSan): name the real cause instead of "build/crash error".
|
||||
if grep -qE "$SAN_RE" "$LOG"; then
|
||||
grep -nE "$SAN_RE" "$LOG" | head -4 | sed 's/^/ /'
|
||||
echo " -> sanitizer fault: if every test above is PASS, this is an exit-time abort, not a failed assertion."
|
||||
echo " -> read the full report by running the binary BARE (gdb hides it via ptrace): ./$bin 2>&1 | tail -40"
|
||||
echo "RESULT: RED sanitizer fault — $(grep -ohE 'SUMMARY: [A-Za-z]+Sanitizer:.*' "$LOG" | tail -1 || echo 'see report above')"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# All tests passed but the process still aborted at EXIT (ERRORED/SIGHUP/SIGABRT) and the
|
||||
# sanitizer report was swallowed by the runner (often surfaced only as SIGHUP). Almost always a
|
||||
# sanitizer fault — point at how to surface it rather than calling it a generic crash.
|
||||
if grep -qE "$PASS_RE" "$LOG" && grep -qE '\[ERRORED\]|SIGHUP|SIGABRT' "$LOG" && ! grep -qE ':FAIL\b|\[FAILED\]' "$LOG"; then
|
||||
echo " -> all tests passed but the process aborted at EXIT — likely an ASan/LSan fault whose report"
|
||||
echo " the runner swallowed (commonly shown as SIGHUP). Run the binary BARE to see it: ./$bin 2>&1 | tail -40"
|
||||
echo "RESULT: RED exit-time abort (tests passed; likely sanitizer — see hint above)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "RESULT: RED $(grep -oE '[0-9]+ failed' "$LOG" | tail -1 || echo 'build/crash error')"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# RED: pio non-zero, any failure marker, or no positive summary at all (build died early).
|
||||
if [[ $PIO_RC -ne 0 ]] || grep -qE "$FAIL_RE" "$LOG"; then
|
||||
verdict_red
|
||||
fi
|
||||
if ! grep -qE "$PASS_RE" "$LOG"; then
|
||||
echo ""
|
||||
echo "RESULT: RED no success summary found (build error / no tests ran?) — see log"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# AMBER: everything that ran passed, but (full run only) a canonical suite neither ran NOR was
|
||||
# explicitly skipped — i.e. it silently went missing. SKIPPED suites are accounted for.
|
||||
ACCOUNTED_COUNT=$((RAN_COUNT + ${#SKIPPED_SUITES[@]}))
|
||||
if [[ -z $FILTER && $ACCOUNTED_COUNT -lt $EXPECTED_COUNT ]]; then
|
||||
missing=()
|
||||
for s in "${ALL_SUITES[@]}"; do
|
||||
printf '%s\n' "${RAN_SUITES[@]}" "${SKIPPED_SUITES[@]}" | grep -qx "$s" || missing+=("$s")
|
||||
done
|
||||
echo ""
|
||||
echo "RESULT: AMBER ${RAN_COUNT}/${EXPECTED_COUNT} suites ran (missing: ${missing[*]}) — all that ran passed"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# GREEN.
|
||||
if [[ -n $FILTER ]]; then
|
||||
echo "RESULT: GREEN ${RAN_COUNT} suite(s) passed (filtered: $FILTER)"
|
||||
else
|
||||
echo "RESULT: GREEN ${RAN_COUNT}/${EXPECTED_COUNT} suites passed"
|
||||
fi
|
||||
exit 0
|
||||
@@ -1,342 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compile a committed seed JSONL into a binary meshtastic_NodeDatabase v25 proto.
|
||||
|
||||
The input is produced by `bin/gen-fake-nodedb-seed.py`. Timestamps in the JSONL
|
||||
are stored as `*_offset_sec` (seconds before "now"); this script resolves them
|
||||
to absolute epochs using `--now-epoch` (default: current wall clock).
|
||||
|
||||
Output is a raw `pb_encode`-compatible binary that can be dropped at
|
||||
`/prefs/nodes.proto` on the device (Portduino prefs dir or hardware via
|
||||
XModem) and loaded by `NodeDB::loadFromDisk` at boot.
|
||||
|
||||
Wire format reference:
|
||||
protobufs/meshtastic/deviceonly.proto (NodeDatabase, NodeInfoLite, sat entries)
|
||||
src/mesh/NodeDB.h:467-484 (bitfield bit positions)
|
||||
src/mesh/NodeDB.cpp:1523-1524 (pb_decode entry point)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
# Prefer the in-tree generated Python protobuf bindings (bin/_generated/meshtastic_v25/)
|
||||
# because the firmware branch's protos (v25 NodeDatabase satellite arrays, slim
|
||||
# NodeInfoLite) are typically newer than what the PyPI `meshtastic` package
|
||||
# ships. Run `bin/regen-py-protos.sh` to (re)generate.
|
||||
#
|
||||
# Namespace note: the local bindings live under `meshtastic_v25` (NOT `meshtastic`)
|
||||
# to avoid shadowing the PyPI `meshtastic` package — bin/regen-py-protos.sh
|
||||
# post-processes the protoc output to rename the package.
|
||||
_HERE = pathlib.Path(__file__).resolve().parent
|
||||
_LOCAL_PROTO_DIR = _HERE / "_generated"
|
||||
if _LOCAL_PROTO_DIR.is_dir():
|
||||
sys.path.insert(0, str(_LOCAL_PROTO_DIR))
|
||||
|
||||
try:
|
||||
from meshtastic_v25.deviceonly_pb2 import ( # type: ignore[import-not-found]
|
||||
NodeDatabase,
|
||||
NodeInfoLite,
|
||||
NodePositionEntry,
|
||||
NodeTelemetryEntry,
|
||||
NodeEnvironmentEntry,
|
||||
NodeStatusEntry,
|
||||
PositionLite,
|
||||
)
|
||||
from meshtastic_v25.mesh_pb2 import HardwareModel, Position, StatusMessage # type: ignore[import-not-found]
|
||||
from meshtastic_v25.config_pb2 import Config # type: ignore[import-not-found]
|
||||
from meshtastic_v25.telemetry_pb2 import DeviceMetrics, EnvironmentMetrics # type: ignore[import-not-found]
|
||||
except ImportError as local_err:
|
||||
# Fall back to the PyPI package if in-tree bindings haven't been generated.
|
||||
# Will fail the v25 assertion below if the PyPI package predates the
|
||||
# satellite-DB schema, but at least gives a clear "run regen-py-protos.sh"
|
||||
# error message instead of an opaque ImportError.
|
||||
try:
|
||||
from meshtastic.protobuf.deviceonly_pb2 import (
|
||||
NodeDatabase,
|
||||
NodeInfoLite,
|
||||
NodePositionEntry,
|
||||
NodeTelemetryEntry,
|
||||
NodeEnvironmentEntry,
|
||||
NodeStatusEntry,
|
||||
PositionLite,
|
||||
)
|
||||
from meshtastic.protobuf.mesh_pb2 import HardwareModel, Position, StatusMessage
|
||||
from meshtastic.protobuf.config_pb2 import Config
|
||||
from meshtastic.protobuf.telemetry_pb2 import DeviceMetrics, EnvironmentMetrics
|
||||
except ImportError as pypi_err:
|
||||
print(
|
||||
"ERROR: could not import meshtastic protobuf bindings.\n"
|
||||
" In-tree generation: run `bin/regen-py-protos.sh` (requires protoc).\n"
|
||||
" PyPI fallback: `pip install meshtastic` (may lag firmware branch).\n"
|
||||
f" local error (meshtastic_v25): {local_err}\n"
|
||||
f" pypi error (meshtastic.protobuf): {pypi_err}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
# Fail loudly if bindings predate v25 (no satellite arrays).
|
||||
assert (
|
||||
hasattr(NodeDatabase, "DESCRIPTOR")
|
||||
and "positions" in NodeDatabase.DESCRIPTOR.fields_by_name
|
||||
), (
|
||||
"Loaded meshtastic bindings are older than v25 (NodeDatabase.positions missing). "
|
||||
"Run `bin/regen-py-protos.sh` against the in-tree protobufs/ submodule."
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bitfield bit positions (mirror src/mesh/NodeDB.h:467-484).
|
||||
# ---------------------------------------------------------------------------
|
||||
BIT_IS_KEY_MANUALLY_VERIFIED = 0
|
||||
BIT_IS_MUTED = 1
|
||||
BIT_VIA_MQTT = 2
|
||||
BIT_IS_FAVORITE = 3
|
||||
BIT_IS_IGNORED = 4
|
||||
BIT_HAS_USER = 5
|
||||
BIT_IS_LICENSED = 6
|
||||
BIT_IS_UNMESSAGABLE = 7
|
||||
BIT_HAS_IS_UNMESSAGABLE = 8
|
||||
|
||||
BITFIELD_LAYOUT = (
|
||||
# JSON key bit position
|
||||
("is_key_manually_verified", BIT_IS_KEY_MANUALLY_VERIFIED),
|
||||
("is_muted", BIT_IS_MUTED),
|
||||
("via_mqtt", BIT_VIA_MQTT),
|
||||
("is_favorite", BIT_IS_FAVORITE),
|
||||
("is_ignored", BIT_IS_IGNORED),
|
||||
("has_user", BIT_HAS_USER),
|
||||
("is_licensed", BIT_IS_LICENSED),
|
||||
("is_unmessagable", BIT_IS_UNMESSAGABLE),
|
||||
("has_is_unmessagable", BIT_HAS_IS_UNMESSAGABLE),
|
||||
)
|
||||
|
||||
|
||||
def _pack_bitfield(bf: dict[str, bool]) -> int:
|
||||
out = 0
|
||||
for key, shift in BITFIELD_LAYOUT:
|
||||
if bf.get(key, False):
|
||||
out |= (1 << shift)
|
||||
return out
|
||||
|
||||
|
||||
def _validate_node(node: dict[str, Any]) -> None:
|
||||
"""Friendly errors so hand-editors get clear feedback."""
|
||||
if "num" not in node or not isinstance(node["num"], str):
|
||||
raise ValueError(f"node missing/invalid 'num' (must be hex string): {node!r}")
|
||||
if "long_name" not in node:
|
||||
raise ValueError(f"node {node['num']}: missing 'long_name'")
|
||||
if len(node["long_name"]) > 24:
|
||||
raise ValueError(
|
||||
f"node {node['num']}: long_name {node['long_name']!r} is "
|
||||
f"{len(node['long_name'])} chars; max 24 (nanopb max_size:25 minus NUL)"
|
||||
)
|
||||
if "short_name" in node:
|
||||
# short_name max_size:5 (incl. NUL) → 4 bytes of content.
|
||||
# Char count is irrelevant — emojis with variation selectors (e.g. ❄️ = 6 B)
|
||||
# would slip past a `len(str) > 4` check. Always measure bytes.
|
||||
b = node["short_name"].encode("utf-8")
|
||||
if len(b) > 4:
|
||||
raise ValueError(
|
||||
f"node {node['num']}: short_name {node['short_name']!r} is "
|
||||
f"{len(b)} bytes UTF-8; max 4 (nanopb max_size:5 minus NUL)"
|
||||
)
|
||||
pk = node.get("public_key_hex", "")
|
||||
if pk and len(pk) != 64:
|
||||
raise ValueError(
|
||||
f"node {node['num']}: public_key_hex must be 64 hex chars or empty; "
|
||||
f"got {len(pk)} chars"
|
||||
)
|
||||
if pk:
|
||||
try:
|
||||
bytes.fromhex(pk)
|
||||
except ValueError as e:
|
||||
raise ValueError(f"node {node['num']}: public_key_hex is not valid hex: {e}")
|
||||
|
||||
|
||||
def _resolve_time(
|
||||
node: dict[str, Any],
|
||||
field_absolute: str,
|
||||
field_offset: str,
|
||||
now_epoch: int,
|
||||
) -> int:
|
||||
"""If `field_absolute` is set, use it; else compute `now_epoch - offset`."""
|
||||
if field_absolute in node and node[field_absolute] is not None:
|
||||
return int(node[field_absolute])
|
||||
offset = node.get(field_offset, 0)
|
||||
return max(0, int(now_epoch) - int(offset))
|
||||
|
||||
|
||||
def _build_node_info_lite(node: dict[str, Any], now_epoch: int) -> NodeInfoLite:
|
||||
_validate_node(node)
|
||||
info = NodeInfoLite()
|
||||
info.num = int(node["num"], 16) if isinstance(node["num"], str) else int(node["num"])
|
||||
info.long_name = node.get("long_name", "")
|
||||
info.short_name = node.get("short_name", "")
|
||||
# Enum lookups will raise ValueError on unknown names — that's exactly what we want.
|
||||
info.hw_model = HardwareModel.Value(node.get("hw_model", "UNSET"))
|
||||
info.role = Config.DeviceConfig.Role.Value(node.get("role", "CLIENT"))
|
||||
pk_hex = node.get("public_key_hex", "")
|
||||
if pk_hex:
|
||||
info.public_key = bytes.fromhex(pk_hex)
|
||||
info.snr = float(node.get("snr", 0.0))
|
||||
info.channel = int(node.get("channel", 0))
|
||||
if "hops_away" in node:
|
||||
# `optional uint32 hops_away = 9;` — in Python protobuf, assigning the
|
||||
# field implicitly sets HasField("hops_away") to True. No has_hops_away
|
||||
# setter exists (unlike the C++ nanopb-generated header).
|
||||
info.hops_away = int(node["hops_away"])
|
||||
info.next_hop = int(node.get("next_hop", 0))
|
||||
info.last_heard = _resolve_time(node, "last_heard", "last_heard_offset_sec", now_epoch)
|
||||
info.bitfield = _pack_bitfield(node.get("bitfield", {}))
|
||||
return info
|
||||
|
||||
|
||||
def _build_position_entry(num: int, pos: dict[str, Any], now_epoch: int) -> NodePositionEntry:
|
||||
entry = NodePositionEntry()
|
||||
entry.num = num
|
||||
pl = PositionLite()
|
||||
# Firmware stores lat/long as int32 in 1e-7 degrees.
|
||||
pl.latitude_i = int(round(float(pos["latitude"]) * 1e7))
|
||||
pl.longitude_i = int(round(float(pos["longitude"]) * 1e7))
|
||||
pl.altitude = int(pos.get("altitude", 0))
|
||||
pl.time = _resolve_time(pos, "time", "time_offset_sec", now_epoch)
|
||||
pl.location_source = Position.LocSource.Value(pos.get("location_source", "LOC_UNSET"))
|
||||
entry.position.CopyFrom(pl)
|
||||
return entry
|
||||
|
||||
|
||||
def _build_telemetry_entry(num: int, tel: dict[str, Any]) -> NodeTelemetryEntry:
|
||||
entry = NodeTelemetryEntry()
|
||||
entry.num = num
|
||||
dm = DeviceMetrics()
|
||||
if "battery_level" in tel:
|
||||
dm.battery_level = int(tel["battery_level"])
|
||||
if "voltage" in tel:
|
||||
dm.voltage = float(tel["voltage"])
|
||||
if "channel_utilization" in tel:
|
||||
dm.channel_utilization = float(tel["channel_utilization"])
|
||||
if "air_util_tx" in tel:
|
||||
dm.air_util_tx = float(tel["air_util_tx"])
|
||||
if "uptime_seconds" in tel:
|
||||
dm.uptime_seconds = int(tel["uptime_seconds"])
|
||||
entry.device_metrics.CopyFrom(dm)
|
||||
return entry
|
||||
|
||||
|
||||
def _build_environment_entry(num: int, env: dict[str, Any]) -> NodeEnvironmentEntry:
|
||||
entry = NodeEnvironmentEntry()
|
||||
entry.num = num
|
||||
em = EnvironmentMetrics()
|
||||
if "temperature" in env:
|
||||
em.temperature = float(env["temperature"])
|
||||
if "relative_humidity" in env:
|
||||
em.relative_humidity = float(env["relative_humidity"])
|
||||
if "barometric_pressure" in env:
|
||||
em.barometric_pressure = float(env["barometric_pressure"])
|
||||
if "iaq" in env:
|
||||
em.iaq = int(env["iaq"])
|
||||
entry.environment_metrics.CopyFrom(em)
|
||||
return entry
|
||||
|
||||
|
||||
def _build_status_entry(num: int, status: dict[str, Any]) -> NodeStatusEntry:
|
||||
# `StatusMessage` (mesh.proto:1445) has a single `string status` field.
|
||||
entry = NodeStatusEntry()
|
||||
entry.num = num
|
||||
sm = StatusMessage()
|
||||
if "status" in status:
|
||||
sm.status = str(status["status"])
|
||||
entry.status.CopyFrom(sm)
|
||||
return entry
|
||||
|
||||
|
||||
def compile_jsonl_to_proto(jsonl_path: pathlib.Path, now_epoch: int) -> bytes:
|
||||
"""Read a seed JSONL and return the encoded NodeDatabase bytes."""
|
||||
lines = jsonl_path.read_text(encoding="utf-8").splitlines()
|
||||
if not lines:
|
||||
raise ValueError(f"{jsonl_path} is empty")
|
||||
meta_line = lines[0]
|
||||
meta_obj = json.loads(meta_line)
|
||||
meta = meta_obj.get("_meta", {})
|
||||
version = meta.get("version")
|
||||
if version != 25:
|
||||
raise ValueError(
|
||||
f"{jsonl_path}: meta version is {version!r}; this compiler "
|
||||
f"requires version=25. Regenerate the seed with the matching tooling."
|
||||
)
|
||||
|
||||
db = NodeDatabase()
|
||||
db.version = 25
|
||||
|
||||
for ln, raw in enumerate(lines[1:], start=2):
|
||||
raw = raw.strip()
|
||||
if not raw:
|
||||
continue
|
||||
try:
|
||||
node = json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError(f"{jsonl_path}:{ln} JSON parse error: {e}")
|
||||
|
||||
num = int(node["num"], 16) if isinstance(node["num"], str) else int(node["num"])
|
||||
|
||||
# Header
|
||||
info = _build_node_info_lite(node, now_epoch)
|
||||
db.nodes.append(info)
|
||||
|
||||
# Satellites (nullable)
|
||||
if node.get("position"):
|
||||
db.positions.append(_build_position_entry(num, node["position"], now_epoch))
|
||||
if node.get("telemetry"):
|
||||
db.telemetry.append(_build_telemetry_entry(num, node["telemetry"]))
|
||||
if node.get("environment"):
|
||||
db.environment.append(_build_environment_entry(num, node["environment"]))
|
||||
if node.get("status"):
|
||||
db.status.append(_build_status_entry(num, node["status"]))
|
||||
|
||||
return db.SerializeToString()
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
p = argparse.ArgumentParser(
|
||||
description="Compile a seed JSONL into a binary v25 NodeDatabase proto.",
|
||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
||||
)
|
||||
p.add_argument("--in", dest="in_path", required=True, help="Input seed JSONL.")
|
||||
p.add_argument("--out", required=True, help="Output binary .proto path.")
|
||||
p.add_argument(
|
||||
"--now-epoch",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Pin 'now' to this Unix epoch (for byte-identical CI). Default: time.time().",
|
||||
)
|
||||
args = p.parse_args(argv)
|
||||
|
||||
in_path = pathlib.Path(args.in_path)
|
||||
if not in_path.is_file():
|
||||
print(f"input not found: {in_path}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
now_epoch = args.now_epoch if args.now_epoch is not None else int(time.time())
|
||||
|
||||
try:
|
||||
encoded = compile_jsonl_to_proto(in_path, now_epoch)
|
||||
except ValueError as e:
|
||||
print(f"ERROR: {e}", file=sys.stderr)
|
||||
return 3
|
||||
|
||||
out_path = pathlib.Path(args.out)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_bytes(encoded)
|
||||
print(
|
||||
f"compiled {in_path} -> {out_path} ({len(encoded)} bytes, now_epoch={now_epoch})",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,95 @@
|
||||
import sys
|
||||
import os
|
||||
import json
|
||||
from github import Github
|
||||
|
||||
def parseFile(path):
|
||||
with open(path, "r") as f:
|
||||
data = json.loads(f)
|
||||
for file in data["files"]:
|
||||
if file["name"].endswith(".bin"):
|
||||
return file["name"], file["bytes"]
|
||||
|
||||
if len(sys.argv) != 4:
|
||||
print(f"expected usage: {sys.argv[0]} <PR number> <path to old-manifests> <path to new-manifests>")
|
||||
sys.exit(1)
|
||||
|
||||
pr_number = int(sys.argv[1])
|
||||
|
||||
token = os.getenv("GITHUB_TOKEN")
|
||||
if not token:
|
||||
raise EnvironmentError("GITHUB_TOKEN not found in environment.")
|
||||
|
||||
repo_name = os.getenv("GITHUB_REPOSITORY") # "owner/repo"
|
||||
if not repo_name:
|
||||
raise EnvironmentError("GITHUB_REPOSITORY not found in environment.")
|
||||
|
||||
oldFiles = sys.argv[2]
|
||||
old = set(os.path.join(oldFiles, f) for f in os.listdir(oldFiles) if os.path.isfile(f))
|
||||
newFiles = sys.argv[3]
|
||||
new = set(os.path.join(newFiles, f) for f in os.listdir(newFiles) if os.path.isfile(f))
|
||||
|
||||
startMarkdown = "# Target Size Changes\n\n"
|
||||
markdown = ""
|
||||
|
||||
newlyIntroduced = new - old
|
||||
if len(newlyIntroduced) > 0:
|
||||
markdown += "## Newly Introduced Targets\n\n"
|
||||
# create a table
|
||||
markdown += "| File | Size |\n"
|
||||
markdown += "| ---- | ---- |\n"
|
||||
for f in newlyIntroduced:
|
||||
name, size = parseFile(f)
|
||||
markdown += f"| `{name}` | {size}b |\n"
|
||||
|
||||
# do not log removed targets
|
||||
# PRs only run a small subset of builds, so removed targets are not meaningful
|
||||
# since they are very likely to just be not ran in PR CI
|
||||
|
||||
both = old & new
|
||||
degradations = []
|
||||
improvements = []
|
||||
for f in both:
|
||||
oldName, oldSize = parseFile(f)
|
||||
_, newSize = parseFile(f)
|
||||
if oldSize != newSize:
|
||||
if newSize < oldSize:
|
||||
improvements.append((oldName, oldSize, newSize))
|
||||
else:
|
||||
degradations.append((oldName, oldSize, newSize))
|
||||
|
||||
if len(degradations) > 0:
|
||||
markdown += "\n## Degradation\n\n"
|
||||
# create a table
|
||||
markdown += "| File | Difference | Old Size | New Size |\n"
|
||||
markdown += "| ---- | ---------- | -------- | -------- |\n"
|
||||
for oldName, oldSize, newSize in degradations:
|
||||
markdown += f"| `{oldName}` | **{oldSize - newSize}b** | {oldSize}b | {newSize}b |\n"
|
||||
|
||||
if len(improvements) > 0:
|
||||
markdown += "\n## Improvement\n\n"
|
||||
# create a table
|
||||
markdown += "| File | Difference | Old Size | New Size |\n"
|
||||
markdown += "| ---- | ---------- | -------- | -------- |\n"
|
||||
for oldName, oldSize, newSize in improvements:
|
||||
markdown += f"| `{oldName}` | **{oldSize - newSize}b** | {oldSize}b | {newSize}b |\n"
|
||||
|
||||
if len(markdown) == 0:
|
||||
markdown = "No changes in target sizes detected."
|
||||
|
||||
g = Github(token)
|
||||
repo = g.get_repo(repo_name)
|
||||
pr = repo.get_pull(pr_number)
|
||||
|
||||
existing_comment = None
|
||||
for comment in pr.get_issue_comments():
|
||||
if comment.body.startswith(startMarkdown):
|
||||
existing_comment = comment
|
||||
break
|
||||
|
||||
final_markdown = startMarkdown + markdown
|
||||
|
||||
if existing_comment:
|
||||
existing_comment.edit(body=final_markdown)
|
||||
else:
|
||||
pr.create_issue_comment(body=final_markdown)
|
||||
@@ -1,171 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""Compare firmware size reports and generate a markdown summary.
|
||||
|
||||
Usage:
|
||||
size_report.py <new_sizes.json> [--baseline <label>:<old_sizes.json>]...
|
||||
|
||||
Examples:
|
||||
# Compare PR against develop and master baselines
|
||||
size_report.py pr.json --baseline develop:develop.json --baseline master:master.json
|
||||
|
||||
# Single baseline comparison
|
||||
size_report.py pr.json --baseline develop:develop.json
|
||||
|
||||
# No baselines — shows sizes with blank delta columns
|
||||
size_report.py pr.json
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
def load_sizes(path):
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def format_delta(n):
|
||||
"""Format byte delta with sign and human-friendly suffix."""
|
||||
sign = "+" if n > 0 else ""
|
||||
if abs(n) >= 1024:
|
||||
return f"{sign}{n:,} ({sign}{n / 1024:.1f} KB)"
|
||||
return f"{sign}{n:,}"
|
||||
|
||||
|
||||
def generate_markdown(new_sizes, baselines, top_n=5):
|
||||
"""Generate a single table with current size and delta columns per baseline.
|
||||
|
||||
baselines: list of (label, old_sizes_dict), may be empty
|
||||
"""
|
||||
labels = [label for label, _ in baselines]
|
||||
|
||||
# Build rows: (board, current_size, [(delta, abs_delta) per baseline])
|
||||
rows = []
|
||||
for board in sorted(new_sizes):
|
||||
current = new_sizes[board]
|
||||
deltas = []
|
||||
for _, old_sizes in baselines:
|
||||
old = old_sizes.get(board)
|
||||
if old is not None:
|
||||
d = current - old
|
||||
deltas.append((d, abs(d)))
|
||||
else:
|
||||
deltas.append((None, 0))
|
||||
# Sort key: max abs delta across baselines (biggest changes first)
|
||||
max_abs = max((ad for _, ad in deltas), default=0)
|
||||
rows.append((board, current, deltas, max_abs))
|
||||
|
||||
rows.sort(key=lambda r: r[3], reverse=True)
|
||||
|
||||
# Summary line
|
||||
sections = []
|
||||
summary_parts = [f"{len(new_sizes)} targets"]
|
||||
for i, (label, old_sizes) in enumerate(baselines):
|
||||
increases = sum(
|
||||
1 for _, _, deltas, _ in rows if deltas[i][0] is not None and deltas[i][0] > 0
|
||||
)
|
||||
decreases = sum(
|
||||
1 for _, _, deltas, _ in rows if deltas[i][0] is not None and deltas[i][0] < 0
|
||||
)
|
||||
net = sum(
|
||||
deltas[i][0] for _, _, deltas, _ in rows if deltas[i][0] is not None
|
||||
)
|
||||
parts = []
|
||||
if increases:
|
||||
parts.append(f"{increases} increased")
|
||||
if decreases:
|
||||
parts.append(f"{decreases} decreased")
|
||||
if parts:
|
||||
parts.append(f"net {format_delta(net)}")
|
||||
summary_parts.append(f"vs `{label}`: {', '.join(parts)}")
|
||||
else:
|
||||
summary_parts.append(f"vs `{label}`: no changes")
|
||||
|
||||
if not baselines:
|
||||
summary_parts.append("no baseline available yet")
|
||||
|
||||
sections.append(f"**{' | '.join(summary_parts)}**\n")
|
||||
|
||||
# Table header
|
||||
header = "| Target | Size |"
|
||||
separator = "|--------|-----:|"
|
||||
for label in labels:
|
||||
header += f" vs `{label}` |"
|
||||
separator += "----------:|"
|
||||
sections.append(header)
|
||||
sections.append(separator)
|
||||
|
||||
def format_row(board, current, deltas):
|
||||
row = f"| `{board}` | {current:,} |"
|
||||
for d, _ in deltas:
|
||||
if d is None:
|
||||
row += " |"
|
||||
elif d == 0:
|
||||
row += " 0 |"
|
||||
else:
|
||||
icon = "📈" if d > 0 else "📉"
|
||||
row += f" {icon} {format_delta(d)} |"
|
||||
return row
|
||||
|
||||
# Top N rows always visible
|
||||
top = rows[:top_n]
|
||||
for board, current, deltas, _ in top:
|
||||
sections.append(format_row(board, current, deltas))
|
||||
|
||||
# Remaining rows in expandable section
|
||||
rest = rows[top_n:]
|
||||
if rest:
|
||||
sections.append("")
|
||||
sections.append(
|
||||
f"<details><summary>Show {len(rest)} more target(s)</summary>\n"
|
||||
)
|
||||
sections.append(header)
|
||||
sections.append(separator)
|
||||
for board, current, deltas, _ in rest:
|
||||
sections.append(format_row(board, current, deltas))
|
||||
sections.append("\n</details>")
|
||||
|
||||
sections.append("")
|
||||
return "\n".join(sections)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Compare firmware size reports")
|
||||
parser.add_argument("new_sizes", help="Path to new sizes JSON")
|
||||
parser.add_argument(
|
||||
"--baseline",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="LABEL:PATH",
|
||||
help="Baseline to compare against (e.g. develop:develop.json)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--top",
|
||||
type=int,
|
||||
default=5,
|
||||
help="Number of top changes to show before collapsing (default: 5)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
new_sizes = load_sizes(args.new_sizes)
|
||||
|
||||
# Silence output when no targets were built — repo maintainer choice
|
||||
if not new_sizes:
|
||||
return
|
||||
|
||||
baselines = []
|
||||
for b in args.baseline:
|
||||
if ":" not in b:
|
||||
print(f"Error: baseline must be LABEL:PATH, got '{b}'", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
label, path = b.split(":", 1)
|
||||
baselines.append((label, load_sizes(path)))
|
||||
|
||||
md = generate_markdown(new_sizes, baselines, top_n=args.top)
|
||||
print(md)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,262 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""Tests for bin/collect_sizes.py and bin/size_report.py."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
SCRIPTS_DIR = os.path.join(os.path.dirname(__file__), "..", "bin")
|
||||
|
||||
|
||||
def make_manifest(target, firmware_bytes, extra_files=None):
|
||||
"""Create a minimal .mt.json manifest dict."""
|
||||
files = [{"name": f"firmware-{target}-2.6.0.bin", "bytes": firmware_bytes}]
|
||||
if extra_files:
|
||||
files.extend(extra_files)
|
||||
return {
|
||||
"platformioTarget": target,
|
||||
"version": "2.6.0.test",
|
||||
"files": files,
|
||||
}
|
||||
|
||||
|
||||
def write_manifests(tmpdir, manifests):
|
||||
"""Write manifest dicts as .mt.json files into tmpdir."""
|
||||
for target, data in manifests.items():
|
||||
path = os.path.join(tmpdir, f"firmware-{target}.mt.json")
|
||||
with open(path, "w") as f:
|
||||
json.dump(data, f)
|
||||
|
||||
|
||||
def run_script(script, args):
|
||||
"""Run a Python script and return (returncode, stdout, stderr)."""
|
||||
result = subprocess.run(
|
||||
[sys.executable, os.path.join(SCRIPTS_DIR, script)] + args,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return result.returncode, result.stdout, result.stderr
|
||||
|
||||
|
||||
def test_collect_sizes_basic():
|
||||
"""collect_sizes picks up firmware-*.bin entries from manifests."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
outfile = os.path.join(tmpdir, "sizes.json")
|
||||
manifests = {
|
||||
"heltec-v3": make_manifest("heltec-v3", 1048576),
|
||||
"rak4631": make_manifest("rak4631", 524288),
|
||||
"tbeam": make_manifest("tbeam", 786432),
|
||||
}
|
||||
write_manifests(tmpdir, manifests)
|
||||
|
||||
rc, stdout, stderr = run_script("collect_sizes.py", [tmpdir, outfile])
|
||||
assert rc == 0, f"collect_sizes failed: {stderr}"
|
||||
assert "3 targets" in stdout
|
||||
|
||||
with open(outfile) as f:
|
||||
sizes = json.load(f)
|
||||
assert sizes == {"heltec-v3": 1048576, "rak4631": 524288, "tbeam": 786432}
|
||||
|
||||
|
||||
def test_collect_sizes_fallback_bin():
|
||||
"""collect_sizes falls back to non-firmware-prefixed .bin if no firmware-*.bin."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
outfile = os.path.join(tmpdir, "sizes.json")
|
||||
# Manifest with only a generic .bin (no firmware- prefix)
|
||||
data = {
|
||||
"platformioTarget": "custom-board",
|
||||
"files": [
|
||||
{"name": "littlefs-custom-board.bin", "bytes": 100000},
|
||||
{"name": "custom-board.bin", "bytes": 500000},
|
||||
],
|
||||
}
|
||||
path = os.path.join(tmpdir, "firmware-custom-board.mt.json")
|
||||
with open(path, "w") as f:
|
||||
json.dump(data, f)
|
||||
|
||||
rc, stdout, stderr = run_script("collect_sizes.py", [tmpdir, outfile])
|
||||
assert rc == 0, f"collect_sizes failed: {stderr}"
|
||||
|
||||
with open(outfile) as f:
|
||||
sizes = json.load(f)
|
||||
assert sizes == {"custom-board": 500000}
|
||||
|
||||
|
||||
def test_collect_sizes_skips_ota_littlefs():
|
||||
"""collect_sizes ignores ota/littlefs/bleota .bin files in fallback."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
outfile = os.path.join(tmpdir, "sizes.json")
|
||||
data = {
|
||||
"platformioTarget": "board-x",
|
||||
"files": [
|
||||
{"name": "littlefs-board-x.bin", "bytes": 100000},
|
||||
{"name": "bleota-board-x.bin", "bytes": 50000},
|
||||
{"name": "mt-board-x-ota.bin", "bytes": 60000},
|
||||
],
|
||||
}
|
||||
path = os.path.join(tmpdir, "firmware-board-x.mt.json")
|
||||
with open(path, "w") as f:
|
||||
json.dump(data, f)
|
||||
|
||||
rc, stdout, stderr = run_script("collect_sizes.py", [tmpdir, outfile])
|
||||
assert rc == 0
|
||||
with open(outfile) as f:
|
||||
sizes = json.load(f)
|
||||
# No valid firmware .bin found, board should be absent
|
||||
assert sizes == {}
|
||||
|
||||
|
||||
def test_collect_sizes_ignores_non_mt_json():
|
||||
"""collect_sizes skips non .mt.json files."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
outfile = os.path.join(tmpdir, "sizes.json")
|
||||
# Write a valid manifest
|
||||
manifests = {"rak4631": make_manifest("rak4631", 500000)}
|
||||
write_manifests(tmpdir, manifests)
|
||||
# Write a decoy file
|
||||
with open(os.path.join(tmpdir, "readme.txt"), "w") as f:
|
||||
f.write("not a manifest")
|
||||
|
||||
rc, stdout, stderr = run_script("collect_sizes.py", [tmpdir, outfile])
|
||||
assert rc == 0
|
||||
with open(outfile) as f:
|
||||
sizes = json.load(f)
|
||||
assert list(sizes.keys()) == ["rak4631"]
|
||||
|
||||
|
||||
def test_size_report_no_baseline():
|
||||
"""size_report with no baselines shows sizes only."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
sizes_file = os.path.join(tmpdir, "new.json")
|
||||
with open(sizes_file, "w") as f:
|
||||
json.dump({"heltec-v3": 1000000, "rak4631": 500000}, f)
|
||||
|
||||
rc, stdout, stderr = run_script("size_report.py", [sizes_file])
|
||||
assert rc == 0, f"size_report failed: {stderr}"
|
||||
assert "2 targets" in stdout
|
||||
assert "no baseline available yet" in stdout
|
||||
assert "`heltec-v3`" in stdout
|
||||
assert "`rak4631`" in stdout
|
||||
|
||||
|
||||
def test_size_report_with_baseline():
|
||||
"""size_report shows deltas against a baseline."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
new_file = os.path.join(tmpdir, "new.json")
|
||||
old_file = os.path.join(tmpdir, "old.json")
|
||||
with open(new_file, "w") as f:
|
||||
json.dump({"heltec-v3": 1050000, "rak4631": 500000, "tbeam": 800000}, f)
|
||||
with open(old_file, "w") as f:
|
||||
json.dump({"heltec-v3": 1000000, "rak4631": 500000, "tbeam": 810000}, f)
|
||||
|
||||
rc, stdout, stderr = run_script(
|
||||
"size_report.py", [new_file, "--baseline", f"develop:{old_file}"]
|
||||
)
|
||||
assert rc == 0, f"size_report failed: {stderr}"
|
||||
assert "3 targets" in stdout
|
||||
assert "1 increased" in stdout
|
||||
assert "1 decreased" in stdout
|
||||
# heltec-v3 grew by 50000
|
||||
assert "📈" in stdout
|
||||
# tbeam shrank by 10000
|
||||
assert "📉" in stdout
|
||||
# rak4631 unchanged
|
||||
assert "vs `develop`" in stdout
|
||||
|
||||
|
||||
def test_size_report_multiple_baselines():
|
||||
"""size_report handles multiple baselines."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
new_file = os.path.join(tmpdir, "new.json")
|
||||
dev_file = os.path.join(tmpdir, "develop.json")
|
||||
master_file = os.path.join(tmpdir, "master.json")
|
||||
with open(new_file, "w") as f:
|
||||
json.dump({"board-a": 100000}, f)
|
||||
with open(dev_file, "w") as f:
|
||||
json.dump({"board-a": 95000}, f)
|
||||
with open(master_file, "w") as f:
|
||||
json.dump({"board-a": 90000}, f)
|
||||
|
||||
rc, stdout, stderr = run_script(
|
||||
"size_report.py",
|
||||
[new_file, "--baseline", f"develop:{dev_file}", "--baseline", f"master:{master_file}"],
|
||||
)
|
||||
assert rc == 0, f"size_report failed: {stderr}"
|
||||
assert "vs `develop`" in stdout
|
||||
assert "vs `master`" in stdout
|
||||
|
||||
|
||||
def test_size_report_new_target_no_baseline_entry():
|
||||
"""size_report handles targets not present in baseline (new boards)."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
new_file = os.path.join(tmpdir, "new.json")
|
||||
old_file = os.path.join(tmpdir, "old.json")
|
||||
with open(new_file, "w") as f:
|
||||
json.dump({"new-board": 300000, "existing": 500000}, f)
|
||||
with open(old_file, "w") as f:
|
||||
json.dump({"existing": 500000}, f)
|
||||
|
||||
rc, stdout, stderr = run_script(
|
||||
"size_report.py", [new_file, "--baseline", f"develop:{old_file}"]
|
||||
)
|
||||
assert rc == 0, f"size_report failed: {stderr}"
|
||||
assert "`new-board`" in stdout
|
||||
assert "no changes" in stdout # only existing is compared, delta=0
|
||||
|
||||
|
||||
def test_size_report_all_unchanged():
|
||||
"""size_report shows 'no changes' when all sizes match."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
sizes_file = os.path.join(tmpdir, "sizes.json")
|
||||
with open(sizes_file, "w") as f:
|
||||
json.dump({"board-a": 100000, "board-b": 200000}, f)
|
||||
|
||||
rc, stdout, stderr = run_script(
|
||||
"size_report.py", [sizes_file, "--baseline", f"develop:{sizes_file}"]
|
||||
)
|
||||
assert rc == 0, f"size_report failed: {stderr}"
|
||||
assert "no changes" in stdout
|
||||
|
||||
|
||||
def test_collect_sizes_bad_args():
|
||||
"""collect_sizes exits with error on wrong arg count."""
|
||||
rc, stdout, stderr = run_script("collect_sizes.py", [])
|
||||
assert rc == 1
|
||||
assert "Usage" in stderr
|
||||
|
||||
|
||||
def test_size_report_bad_baseline_format():
|
||||
"""size_report exits with error on malformed --baseline."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
sizes_file = os.path.join(tmpdir, "sizes.json")
|
||||
with open(sizes_file, "w") as f:
|
||||
json.dump({"x": 1}, f)
|
||||
|
||||
rc, stdout, stderr = run_script(
|
||||
"size_report.py", [sizes_file, "--baseline", "no-colon-here"]
|
||||
)
|
||||
assert rc == 1
|
||||
assert "LABEL:PATH" in stderr
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
tests = [v for k, v in globals().items() if k.startswith("test_")]
|
||||
passed = 0
|
||||
failed = 0
|
||||
for test in tests:
|
||||
try:
|
||||
test()
|
||||
print(f" PASS: {test.__name__}")
|
||||
passed += 1
|
||||
except AssertionError as e:
|
||||
print(f" FAIL: {test.__name__}: {e}")
|
||||
failed += 1
|
||||
except Exception as e:
|
||||
print(f" ERROR: {test.__name__}: {type(e).__name__}: {e}")
|
||||
failed += 1
|
||||
|
||||
print(f"\n{passed} passed, {failed} failed out of {passed + failed}")
|
||||
sys.exit(1 if failed else 0)
|
||||
@@ -7,7 +7,7 @@
|
||||
"extra_flags": [
|
||||
"-D CDEBYTE_EORA_S3",
|
||||
"-D ARDUINO_USB_CDC_ON_BOOT=1",
|
||||
"-D ARDUINO_USB_MODE=1",
|
||||
"-D ARDUINO_USB_MODE=0",
|
||||
"-D ARDUINO_RUNNING_CORE=1",
|
||||
"-D ARDUINO_EVENT_RUNNING_CORE=1",
|
||||
"-D BOARD_HAS_PSRAM"
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"core": "esp32",
|
||||
"extra_flags": [
|
||||
"-DARDUINO_USB_CDC_ON_BOOT=1",
|
||||
"-DARDUINO_USB_MODE=1",
|
||||
"-DARDUINO_USB_MODE=0",
|
||||
"-DARDUINO_RUNNING_CORE=1",
|
||||
"-DARDUINO_EVENT_RUNNING_CORE=1",
|
||||
"-DBOARD_HAS_PSRAM"
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
"extra_flags": [
|
||||
"-DBOARD_HAS_PSRAM",
|
||||
"-DARDUINO_USB_CDC_ON_BOOT=1",
|
||||
"-DARDUINO_USB_MODE=1",
|
||||
"-DARDUINO_USB_MODE=0",
|
||||
"-DARDUINO_RUNNING_CORE=1",
|
||||
"-DARDUINO_EVENT_RUNNING_CORE=1"
|
||||
],
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
{
|
||||
"build": {
|
||||
"arduino": {
|
||||
"ldscript": "nrf52840_s140_v6.ld"
|
||||
},
|
||||
"core": "nRF5",
|
||||
"cpu": "cortex-m4",
|
||||
"extra_flags": "-DNRF52840_XXAA",
|
||||
"f_cpu": "64000000L",
|
||||
"hwids": [
|
||||
["0x239A", "0x4405"],
|
||||
["0x239A", "0x0029"],
|
||||
["0x239A", "0x002A"],
|
||||
["0x2886", "0x1667"]
|
||||
],
|
||||
"usb_product": "HT-n5262",
|
||||
"mcu": "nrf52840",
|
||||
"variant": "heltec_mesh_node_t1",
|
||||
"variants_dir": "variants",
|
||||
"bsp": {
|
||||
"name": "adafruit"
|
||||
},
|
||||
"softdevice": {
|
||||
"sd_flags": "-DS140",
|
||||
"sd_name": "s140",
|
||||
"sd_version": "6.1.1",
|
||||
"sd_fwid": "0x00B6"
|
||||
},
|
||||
"bootloader": {
|
||||
"settings_addr": "0xFF000"
|
||||
}
|
||||
},
|
||||
"connectivity": ["bluetooth"],
|
||||
"debug": {
|
||||
"jlink_device": "nRF52840_xxAA",
|
||||
"onboard_tools": ["jlink"],
|
||||
"svd_path": "nrf52840.svd",
|
||||
"openocd_target": "nrf52840-mdk-rs"
|
||||
},
|
||||
"frameworks": ["arduino"],
|
||||
"name": "Heltec Mesh Node T1",
|
||||
"upload": {
|
||||
"maximum_ram_size": 248832,
|
||||
"maximum_size": 815104,
|
||||
"speed": 115200,
|
||||
"protocol": "nrfutil",
|
||||
"protocols": ["jlink", "nrfjprog", "nrfutil", "stlink"],
|
||||
"use_1200bps_touch": true,
|
||||
"require_upload_port": true,
|
||||
"wait_for_upload_port": true
|
||||
},
|
||||
"url": "https://heltec.org",
|
||||
"vendor": "Heltec"
|
||||
}
|
||||
@@ -9,7 +9,7 @@
|
||||
"extra_flags": [
|
||||
"-DBOARD_HAS_PSRAM",
|
||||
"-DARDUINO_USB_CDC_ON_BOOT=1",
|
||||
"-DARDUINO_USB_MODE=1",
|
||||
"-DARDUINO_USB_MODE=0",
|
||||
"-DARDUINO_RUNNING_CORE=1",
|
||||
"-DARDUINO_EVENT_RUNNING_CORE=1"
|
||||
],
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
"extra_flags": [
|
||||
"-DBOARD_HAS_PSRAM",
|
||||
"-DARDUINO_USB_CDC_ON_BOOT=1",
|
||||
"-DARDUINO_USB_MODE=1",
|
||||
"-DARDUINO_USB_MODE=0",
|
||||
"-DARDUINO_RUNNING_CORE=1",
|
||||
"-DARDUINO_EVENT_RUNNING_CORE=1"
|
||||
],
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
"extra_flags": [
|
||||
"-DBOARD_HAS_PSRAM",
|
||||
"-DARDUINO_USB_CDC_ON_BOOT=1",
|
||||
"-DARDUINO_USB_MODE=1",
|
||||
"-DARDUINO_USB_MODE=0",
|
||||
"-DARDUINO_RUNNING_CORE=1",
|
||||
"-DARDUINO_EVENT_RUNNING_CORE=1"
|
||||
],
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
"extra_flags": [
|
||||
"-DHELTEC_WIRELESS_TRACKER",
|
||||
"-DARDUINO_USB_CDC_ON_BOOT=1",
|
||||
"-DARDUINO_USB_MODE=1",
|
||||
"-DARDUINO_USB_MODE=0",
|
||||
"-DARDUINO_RUNNING_CORE=1",
|
||||
"-DARDUINO_EVENT_RUNNING_CORE=1"
|
||||
],
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
"core": "esp32",
|
||||
"extra_flags": [
|
||||
"-DARDUINO_USB_CDC_ON_BOOT=1",
|
||||
"-DARDUINO_USB_MODE=1",
|
||||
"-DARDUINO_USB_MODE=0",
|
||||
"-DARDUINO_RUNNING_CORE=1",
|
||||
"-DARDUINO_EVENT_RUNNING_CORE=1"
|
||||
],
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
{
|
||||
"build": {
|
||||
"cpu": "cortex-m33",
|
||||
"f_cpu": "128000000L",
|
||||
"mcu": "nrf54l15",
|
||||
"zephyr": {
|
||||
"variant": "nrf54l15dk/nrf54l15/cpuapp"
|
||||
}
|
||||
},
|
||||
"connectivity": ["bluetooth"],
|
||||
"debug": {
|
||||
"default_tools": ["jlink"],
|
||||
"jlink_device": "nRF54L15_M33",
|
||||
"svd_path": "nrf54l15.svd"
|
||||
},
|
||||
"frameworks": ["zephyr"],
|
||||
"name": "Nordic nRF54L15-DK (PCA10156)",
|
||||
"upload": {
|
||||
"maximum_ram_size": 262144,
|
||||
"maximum_size": 1572864,
|
||||
"protocol": "jlink",
|
||||
"protocols": ["jlink"]
|
||||
},
|
||||
"url": "https://www.nordicsemi.com/Products/nRF54L15",
|
||||
"vendor": "Nordic Semiconductor"
|
||||
}
|
||||
@@ -8,7 +8,7 @@
|
||||
"extra_flags": [
|
||||
"-DBOARD_HAS_PSRAM",
|
||||
"-DARDUINO_USB_CDC_ON_BOOT=1",
|
||||
"-DARDUINO_USB_MODE=1",
|
||||
"-DARDUINO_USB_MODE=0",
|
||||
"-DARDUINO_RUNNING_CORE=1",
|
||||
"-DARDUINO_EVENT_RUNNING_CORE=0"
|
||||
],
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
{
|
||||
"build": {
|
||||
"arduino": {
|
||||
"ldscript": "esp32s3_out.ld",
|
||||
"memory_type": "qio_opi"
|
||||
},
|
||||
"core": "esp32",
|
||||
"extra_flags": [
|
||||
"-DBOARD_HAS_PSRAM",
|
||||
"-DARDUINO_USB_CDC_ON_BOOT=1",
|
||||
"-DARDUINO_USB_MODE=1",
|
||||
"-DARDUINO_RUNNING_CORE=1",
|
||||
"-DARDUINO_EVENT_RUNNING_CORE=0"
|
||||
],
|
||||
"f_cpu": "240000000L",
|
||||
"f_flash": "80000000L",
|
||||
"flash_mode": "qio",
|
||||
"hwids": [["0x303A", "0x1001"]],
|
||||
"mcu": "esp32s3",
|
||||
"variant": "station-g3"
|
||||
},
|
||||
"connectivity": ["wifi", "bluetooth", "lora"],
|
||||
"debug": {
|
||||
"default_tool": "esp-builtin",
|
||||
"onboard_tools": ["esp-builtin"],
|
||||
"openocd_target": "esp32s3.cfg"
|
||||
},
|
||||
"frameworks": ["arduino", "espidf"],
|
||||
"name": "BQ Station G3",
|
||||
"upload": {
|
||||
"flash_size": "16MB",
|
||||
"maximum_ram_size": 327680,
|
||||
"maximum_size": 16777216,
|
||||
"use_1200bps_touch": true,
|
||||
"wait_for_upload_port": true,
|
||||
"require_upload_port": true,
|
||||
"speed": 921600
|
||||
},
|
||||
"url": "",
|
||||
"vendor": "BQ Consulting"
|
||||
}
|
||||
@@ -9,7 +9,7 @@
|
||||
"-DBOARD_HAS_PSRAM",
|
||||
"-DLILYGO_TBEAM_1W",
|
||||
"-DARDUINO_USB_CDC_ON_BOOT=1",
|
||||
"-DARDUINO_USB_MODE=1",
|
||||
"-DARDUINO_USB_MODE=0",
|
||||
"-DARDUINO_RUNNING_CORE=1",
|
||||
"-DARDUINO_EVENT_RUNNING_CORE=1"
|
||||
],
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@
|
||||
"extra_flags": [
|
||||
"-DBOARD_HAS_PSRAM",
|
||||
"-DARDUINO_USB_CDC_ON_BOOT=1",
|
||||
"-DARDUINO_USB_MODE=1",
|
||||
"-DARDUINO_USB_MODE=0",
|
||||
"-DARDUINO_RUNNING_CORE=1",
|
||||
"-DARDUINO_EVENT_RUNNING_CORE=1"
|
||||
],
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
"-DBOARD_HAS_PSRAM",
|
||||
"-DLILYGO_TBEAM_S3_CORE",
|
||||
"-DARDUINO_USB_CDC_ON_BOOT=1",
|
||||
"-DARDUINO_USB_MODE=1",
|
||||
"-DARDUINO_USB_MODE=0",
|
||||
"-DARDUINO_RUNNING_CORE=1",
|
||||
"-DARDUINO_EVENT_RUNNING_CORE=1"
|
||||
],
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
"extra_flags": [
|
||||
"-DLILYGO_T3S3_V1",
|
||||
"-DARDUINO_USB_CDC_ON_BOOT=1",
|
||||
"-DARDUINO_USB_MODE=1",
|
||||
"-DARDUINO_USB_MODE=0",
|
||||
"-DARDUINO_RUNNING_CORE=1",
|
||||
"-DARDUINO_EVENT_RUNNING_CORE=1",
|
||||
"-DBOARD_HAS_PSRAM"
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@
|
||||
"-DBOARD_HAS_PSRAM",
|
||||
"-DUNPHONE_SPIN=9",
|
||||
"-DARDUINO_USB_CDC_ON_BOOT=1",
|
||||
"-DARDUINO_USB_MODE=1",
|
||||
"-DARDUINO_USB_MODE=0",
|
||||
"-DARDUINO_RUNNING_CORE=1",
|
||||
"-DARDUINO_EVENT_RUNNING_CORE=1"
|
||||
],
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"build": {
|
||||
"arduino": {
|
||||
"ldscript": "nrf52832_s132_v6.ld"
|
||||
},
|
||||
"core": "nRF5",
|
||||
"cpu": "cortex-m4",
|
||||
"extra_flags": "-DNRF52832_XXAA -DNRF52",
|
||||
"f_cpu": "64000000L",
|
||||
"hwids": [
|
||||
["0x239A", "0x8029"],
|
||||
["0x239A", "0x0029"],
|
||||
["0x239A", "0x002A"],
|
||||
["0x239A", "0x802A"]
|
||||
],
|
||||
"usb_product": "Feather nRF52832 Express",
|
||||
"mcu": "nrf52832",
|
||||
"variant": "WisCore_RAK4600_Board",
|
||||
"bsp": {
|
||||
"name": "adafruit"
|
||||
},
|
||||
"softdevice": {
|
||||
"sd_flags": "-DS132",
|
||||
"sd_name": "s132",
|
||||
"sd_version": "6.1.1",
|
||||
"sd_fwid": "0x00B7"
|
||||
},
|
||||
"zephyr": {
|
||||
"variant": "nrf52_adafruit_feather"
|
||||
}
|
||||
},
|
||||
"connectivity": ["bluetooth"],
|
||||
"debug": {
|
||||
"jlink_device": "nRF52832_xxAA",
|
||||
"svd_path": "nrf52.svd",
|
||||
"openocd_target": "nrf52840-mdk-rs"
|
||||
},
|
||||
"frameworks": ["arduino", "zephyr"],
|
||||
"name": "Adafruit Bluefruit nRF52832 Feather",
|
||||
"upload": {
|
||||
"maximum_ram_size": 65536,
|
||||
"maximum_size": 524288,
|
||||
"require_upload_port": true,
|
||||
"speed": 115200,
|
||||
"protocol": "nrfutil",
|
||||
"protocols": ["jlink", "nrfjprog", "nrfutil", "stlink"]
|
||||
},
|
||||
"url": "https://www.adafruit.com/product/3406",
|
||||
"vendor": "Adafruit"
|
||||
}
|
||||
Vendored
-1
@@ -14,7 +14,6 @@ Build-Depends: debhelper-compat (= 13),
|
||||
g++,
|
||||
pkg-config,
|
||||
libyaml-cpp-dev,
|
||||
libjsoncpp-dev,
|
||||
libgpiod-dev,
|
||||
libbluetooth-dev,
|
||||
libusb-1.0-0-dev,
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
# Name, Type, SubType, Offset, Size, Flags
|
||||
nvs, data, nvs, 0x9000, 0x5000,
|
||||
otadata, data, ota, 0xe000, 0x2000,
|
||||
app0, app, ota_0, 0x10000, 0x640000,
|
||||
app1, app, ota_1, 0x650000,0x640000,
|
||||
spiffs, data, spiffs, 0xc90000,0x360000,
|
||||
coredump, data, coredump,0xFF0000,0x10000,
|
||||
|
@@ -1,7 +0,0 @@
|
||||
# Name, Type, SubType, Offset, Size, Flags
|
||||
nvs, data, nvs, 0x9000, 0x5000,
|
||||
otadata, data, ota, 0xe000, 0x2000,
|
||||
app0, app, ota_0, 0x10000, 0x330000,
|
||||
app1, app, ota_1, 0x340000,0x330000,
|
||||
spiffs, data, spiffs, 0x670000,0x180000,
|
||||
coredump, data, coredump,0x7F0000,0x10000,
|
||||
|
@@ -70,6 +70,17 @@ def esp32_create_combined_bin(source, target, env):
|
||||
|
||||
env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", esp32_create_combined_bin)
|
||||
|
||||
# Enable Newlib Nano formatting to save space
|
||||
# ...but allow printf float support (compromise)
|
||||
env.Append(LINKFLAGS=["--specs=nano.specs", "-u", "_printf_float"])
|
||||
esp32_kind = env.GetProjectOption("custom_esp32_kind")
|
||||
if esp32_kind == "esp32":
|
||||
# Free up some IRAM by removing auxiliary SPI flash chip drivers.
|
||||
# Wrapped stub symbols are defined in src/platform/esp32/iram-quirk.c.
|
||||
env.Append(
|
||||
LINKFLAGS=[
|
||||
"-Wl,--wrap=esp_flash_chip_gd",
|
||||
"-Wl,--wrap=esp_flash_chip_issi",
|
||||
"-Wl,--wrap=esp_flash_chip_winbond",
|
||||
]
|
||||
)
|
||||
else:
|
||||
# For newer ESP32 targets, using newlib nano works better.
|
||||
env.Append(LINKFLAGS=["--specs=nano.specs", "-u", "_printf_float"])
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# trunk-ignore-all(ruff/F821)
|
||||
# trunk-ignore-all(flake8/F821): For SConstruct imports
|
||||
|
||||
# force linker response file instead of command line arguments
|
||||
|
||||
Import("env")
|
||||
|
||||
|
||||
def wrap_with_tempfile(command_key):
|
||||
command = env.get(command_key)
|
||||
if not command or not isinstance(command, str):
|
||||
return
|
||||
if "TEMPFILE(" in command:
|
||||
return
|
||||
env.Replace(**{command_key: "${TEMPFILE('%s')}" % command})
|
||||
|
||||
|
||||
# Force SCons to spill long commands into response files on this target.
|
||||
env.Replace(MAXLINELENGTH=8192)
|
||||
|
||||
for key in ("LINKCOM", "CXXLINKCOM", "SHLINKCOM", "SHCXXLINKCOM"):
|
||||
wrap_with_tempfile(key)
|
||||
@@ -1,148 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# trunk-ignore-all(ruff/F821)
|
||||
# trunk-ignore-all(flake8/F821)
|
||||
#
|
||||
# Whole-image LTO for nrf52840 (~-60KB; ~-23KB beyond src-only LTO), EXCEPT the objects
|
||||
# that own interrupt/exception handlers.
|
||||
#
|
||||
# Every ISR is referenced only from the assembly vector table (gcc_startup_nrf52840.S),
|
||||
# which LTO cannot see -> whole-program LTO judges the handlers dead, removes them, and
|
||||
# the weak `b .` Default_Handler stubs prevail -> the IRQ lands in an infinite loop and the
|
||||
# chip hangs (or the peripheral silently stalls). Compiling the handler-bearing objects
|
||||
# WITHOUT LTO lets ordinary linking keep the strong handlers; everything else stays LTO'd:
|
||||
# - framework core (/FrameworkArduino/, /cores/nRF5/): every nrfx ISR + the FreeRTOS
|
||||
# SVC/PendSV port.
|
||||
# - TinyUSB nrf port (Adafruit_TinyUSB_nrf.cpp): USBD_IRQHandler (USB data path).
|
||||
# - library .cpp files that own a vector ISR (would otherwise be silently dropped):
|
||||
# bluefruit.cpp -> SD_EVT/SWI2_EGU2 (SoftDevice BLE-event delivery -- advertising
|
||||
# hangs without it)
|
||||
# Wire_nRF52.cpp -> SPIM0/TWIM0 + SPIM1/TWIM1 (interrupt-driven I2C/SPI)
|
||||
# PDM.cpp -> PDM_IRQHandler (PDM microphone)
|
||||
# RotaryEncoder.cpp -> QDEC_IRQHandler (hardware quadrature/rotary encoder)
|
||||
#
|
||||
# A post-link guard (bottom of this file) fails the build if a critical handler was dropped
|
||||
# anyway -- so a future deps bump or a new ISR-owning library becomes a red build, not a field
|
||||
# hang. To hunt a dropped ISR by hand: nm the .elf for `_IRQHandler$` symbols marked `W`, then
|
||||
# grep the libs/framework for who defines them.
|
||||
#
|
||||
# HW-validated: RAK4631 (SX1262) + muzi-base (LR1121).
|
||||
import glob
|
||||
import os
|
||||
|
||||
Import("env")
|
||||
|
||||
env.Append(LINKFLAGS=["-flto", "-flto-partition=1to1"])
|
||||
|
||||
# The -fno-lto re-compiles below run with the global env, which lacks the framework's
|
||||
# bundled-library include dirs -- and those libs cross-include each other (Wire pulls in
|
||||
# Adafruit_TinyUSB.h, which pulls in SPI.h, ...). Add every bundled-lib dir (+ its src/) so
|
||||
# the re-compiles resolve without chasing headers one at a time.
|
||||
_fw = env.PioPlatform().get_package_dir("framework-arduinoadafruitnrf52") or ""
|
||||
_extra_inc = []
|
||||
for _d in sorted(glob.glob(os.path.join(_fw, "libraries", "*"))):
|
||||
if os.path.isdir(_d):
|
||||
_extra_inc.append(_d)
|
||||
if os.path.isdir(os.path.join(_d, "src")):
|
||||
_extra_inc.append(os.path.join(_d, "src"))
|
||||
|
||||
FRAMEWORK = ("/FrameworkArduino/", "/cores/nRF5/")
|
||||
USB_ISR = "Adafruit_TinyUSB_nrf" # USBD_IRQHandler
|
||||
# Library .cpp files that define vector-table ISRs (the rest of their lib stays LTO'd):
|
||||
LIB_ISR = ("/bluefruit.cpp", "/Wire_nRF52.cpp", "/PDM.cpp", "/RotaryEncoder.cpp")
|
||||
|
||||
|
||||
def _no_lto(node):
|
||||
try:
|
||||
path = node.get_abspath()
|
||||
except Exception:
|
||||
path = str(node)
|
||||
path = path.replace(
|
||||
"\\", "/"
|
||||
) # normalize Windows backslashes so matches work cross-platform
|
||||
if (
|
||||
USB_ISR in path
|
||||
or any(s in path for s in FRAMEWORK)
|
||||
or any(s in path for s in LIB_ISR)
|
||||
):
|
||||
return env.Object(
|
||||
node,
|
||||
CCFLAGS=env["CCFLAGS"] + ["-fno-lto"],
|
||||
CPPPATH=env["CPPPATH"] + _extra_inc,
|
||||
)
|
||||
return node
|
||||
|
||||
|
||||
env.AddBuildMiddleware(_no_lto)
|
||||
|
||||
|
||||
# --- post-link guard: catch a dropped ISR handler at build time (CI footgun protection) ----
|
||||
# After every link, fail the build if one of these critical vector-table handlers resolved to
|
||||
# the weak `b .` Default_Handler stub -- i.e. LTO (or a deps bump, or a new ISR-owning library
|
||||
# that nobody added to LIB_ISR) silently dropped it. A dropped handler hangs the chip the
|
||||
# instant that IRQ fires; this turns a field hang into a red build. CI builds every nrf52840
|
||||
# target, so this runs on every PR automatically. If a board deliberately stops using one of
|
||||
# these, edit the tuples on purpose.
|
||||
_REQUIRED_STRONG = (
|
||||
"SWI2_EGU2_IRQHandler", # SoftDevice BLE event (SD_EVT) -- advertising & connections
|
||||
"GPIOTE_IRQHandler", # GPIO interrupts: radio DIO + buttons
|
||||
"RTC1_IRQHandler", # FreeRTOS scheduler tick
|
||||
)
|
||||
# Owned by the TinyUSB stack, so only required when the board builds with USB at all.
|
||||
# Boards without native USB wiring (e.g. wio-sdk-wm1110's CH340 UART) strip TinyUSB via
|
||||
# disable_adafruit_usb.py / unflagging USE_TINYUSB, leaving these legitimately weak.
|
||||
_REQUIRED_STRONG_USB = (
|
||||
"USBD_IRQHandler", # USB CDC (serial console + 1200bps DFU trigger)
|
||||
"POWER_CLOCK_IRQHandler", # USB power events (VBUS detect/ready) via TinyUSB hal
|
||||
)
|
||||
|
||||
_tc = env.PioPlatform().get_package_dir("toolchain-gccarmnoneeabi") or ""
|
||||
_NM = os.path.join(_tc, "bin", "arm-none-eabi-nm")
|
||||
if not os.path.isfile(_NM):
|
||||
_NM = "arm-none-eabi-nm" # fall back to PATH
|
||||
|
||||
|
||||
def _assert_isr_handlers_survived(source, target, env):
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
try:
|
||||
# Resolve the ELF at build time; target[0] is the buildprog alias, not the file.
|
||||
elf = env.subst("$BUILD_DIR/${PROGNAME}.elf")
|
||||
out = subprocess.check_output([_NM, elf], universal_newlines=True)
|
||||
except Exception as exc: # tooling hiccup: warn loudly, don't wedge the build
|
||||
print("nrf52_lto: WARNING - ISR-handler guard skipped (nm failed: %s)" % exc)
|
||||
return
|
||||
# nm line: "<addr> <type> <symbol>". type 'T'/'t' = strong (good); 'W'/'w' = weak stub.
|
||||
kind = {}
|
||||
for line in out.split("\n"):
|
||||
f = line.split()
|
||||
if len(f) >= 3 and f[-1].endswith("_IRQHandler"):
|
||||
kind[f[-1]] = f[-2]
|
||||
required = list(_REQUIRED_STRONG)
|
||||
defines = [
|
||||
str(d[0] if isinstance(d, tuple) else d) for d in env.get("CPPDEFINES", [])
|
||||
]
|
||||
if "USE_TINYUSB" in defines:
|
||||
required += _REQUIRED_STRONG_USB
|
||||
dropped = [h for h in required if kind.get(h, "W").upper() != "T"]
|
||||
if dropped:
|
||||
sys.stderr.write(
|
||||
"\n*** nrf52 LTO guard: interrupt handler(s) DROPPED: %s ***\n"
|
||||
"Each resolved to the weak Default_Handler stub, so the chip hangs when that IRQ\n"
|
||||
"fires. Compile the .cpp that defines the handler with -fno-lto by adding it to\n"
|
||||
"LIB_ISR in extra_scripts/nrf52_lto.py. Find the owner of FOO_IRQHandler with:\n"
|
||||
" grep -rl FOO_IRQHandler <framework-arduinoadafruitnrf52>/{libraries,cores}\n\n"
|
||||
% ", ".join(dropped)
|
||||
)
|
||||
from SCons.Script import Exit
|
||||
|
||||
Exit(1) # canonical SCons build-abort -> red build
|
||||
print(
|
||||
"nrf52_lto: ISR-handler guard OK -- %d critical handlers strong" % len(required)
|
||||
)
|
||||
|
||||
|
||||
# Attach to the phony "buildprog" alias, NOT the .elf file node: SCons can skip a post-action
|
||||
# on a file target during an incremental relink (observed), but the buildprog alias runs every
|
||||
# build -- so the guard fires on local incremental rebuilds and clean CI builds alike.
|
||||
env.AddPostAction("buildprog", _assert_isr_handlers_survived)
|
||||
@@ -1,140 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# trunk-ignore-all(ruff/F821)
|
||||
# trunk-ignore-all(flake8/F821): For SConstruct imports
|
||||
#
|
||||
# post:extra_scripts/nrf54l15_linker.py
|
||||
#
|
||||
# Fix for Zephyr two-pass link on nRF54L15:
|
||||
# platformio-build.py registers env.Depends("$PROG_PATH", final_ld_script) but
|
||||
# the SCons dependency chain is broken (final_ld_script Command never runs).
|
||||
# This script adds a PreAction on the final firmware binary that runs the gcc
|
||||
# preprocessing command directly (extracted from build.ninja) to generate
|
||||
# zephyr/linker.cmd before the link step.
|
||||
#
|
||||
# PlatformIO bundles an old Ninja that can't handle multi-output depslog rules,
|
||||
# so we parse the COMMAND line from build.ninja and run just the gcc -E part,
|
||||
# skipping the cmake_transform_depfile step (only needed for Ninja deps tracking).
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
Import("env")
|
||||
|
||||
if env.get("PIOENV") != "nrf54l15dk":
|
||||
pass # Only for the nrf54l15dk environment
|
||||
else:
|
||||
|
||||
def _extract_gcc_command(ninja_build):
|
||||
"""Parse build.ninja to find the gcc -E command that generates linker.cmd.
|
||||
|
||||
The rule format depends on the host:
|
||||
Windows (CMake's RunCMake wraps every command):
|
||||
COMMAND = cmd.exe /C "cd /D DIR && arm-none-eabi-gcc.exe ... -o linker.cmd && cmake.exe -E cmake_transform_depfile ..."
|
||||
POSIX (Linux/macOS — no wrapper):
|
||||
COMMAND = cd DIR && arm-none-eabi-gcc ... -o linker.cmd && cmake -E cmake_transform_depfile ...
|
||||
|
||||
Returns (gcc_cmd_string, cwd_path) or raises RuntimeError.
|
||||
"""
|
||||
in_rule = False
|
||||
with open(ninja_build, "r", encoding="utf-8", errors="replace") as f:
|
||||
for line in f:
|
||||
# Detect start of the linker.cmd custom command rule
|
||||
if not in_rule:
|
||||
if "build zephyr/linker.cmd" in line and "CUSTOM_COMMAND" in line:
|
||||
in_rule = True
|
||||
continue
|
||||
|
||||
stripped = line.strip()
|
||||
if not stripped.startswith("COMMAND = "):
|
||||
continue
|
||||
|
||||
command_val = stripped[len("COMMAND = ") :]
|
||||
|
||||
# On Windows the value is wrapped in `cmd.exe /C "..."` — strip
|
||||
# the wrapper. On POSIX hosts the inner sequence is the value
|
||||
# itself (no quoting layer).
|
||||
m = re.search(r'/C\s+"(.*)"\s*$', command_val)
|
||||
inner = m.group(1) if m else command_val
|
||||
parts = inner.split(" && ")
|
||||
|
||||
cwd = None
|
||||
gcc_cmd = None
|
||||
for part in parts:
|
||||
part = part.strip()
|
||||
if part.startswith("cd /D "): # Windows form
|
||||
cwd = part[len("cd /D ") :]
|
||||
elif part.startswith("cd "): # POSIX form
|
||||
cwd = part[len("cd ") :]
|
||||
elif "arm-none-eabi-gcc" in part:
|
||||
gcc_cmd = part
|
||||
|
||||
if not gcc_cmd:
|
||||
raise RuntimeError(
|
||||
"nRF54L15 linker fix: arm-none-eabi-gcc command not found in:\n%s"
|
||||
% inner[:400]
|
||||
)
|
||||
|
||||
return gcc_cmd, cwd
|
||||
|
||||
raise RuntimeError(
|
||||
"nRF54L15 linker fix: 'build zephyr/linker.cmd' rule not found in build.ninja"
|
||||
)
|
||||
|
||||
def _generate_linker_cmd(target, source, env):
|
||||
"""Generate zephyr/linker.cmd via direct gcc invocation before the final link."""
|
||||
build_dir = env.subst("$BUILD_DIR")
|
||||
zephyr_dir = os.path.join(build_dir, "zephyr")
|
||||
linker_cmd = os.path.join(zephyr_dir, "linker.cmd")
|
||||
|
||||
if os.path.exists(linker_cmd):
|
||||
return # Already present — nothing to do
|
||||
|
||||
ninja_build = os.path.join(build_dir, "build.ninja")
|
||||
if not os.path.exists(ninja_build):
|
||||
raise RuntimeError(
|
||||
"nRF54L15 linker fix: build.ninja not found at %s\n"
|
||||
"Run a full build first so CMake generates the Ninja files."
|
||||
% ninja_build
|
||||
)
|
||||
|
||||
gcc_cmd, cwd = _extract_gcc_command(ninja_build)
|
||||
run_cwd = cwd if cwd else zephyr_dir
|
||||
|
||||
print(
|
||||
"==> nRF54L15: Generating zephyr/linker.cmd (LINKER_ZEPHYR_FINAL) via GCC"
|
||||
)
|
||||
# gcc_cmd comes verbatim from our own build.ninja (never user input) and
|
||||
# contains Windows-style paths with spaces that cannot be safely argv-split
|
||||
# with shlex, so we run it via the platform shell. nosec/nosemgrep below
|
||||
# acknowledge this deliberate, scoped use of shell=True.
|
||||
result = subprocess.run( # nosec B602
|
||||
gcc_cmd,
|
||||
shell=True, # nosemgrep: python.lang.security.audit.subprocess-shell-true.subprocess-shell-true
|
||||
cwd=run_cwd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
print("GCC stdout:", result.stdout[:2000])
|
||||
print("GCC stderr:", result.stderr[:2000])
|
||||
raise RuntimeError(
|
||||
"nRF54L15 linker fix: GCC failed to generate linker.cmd (rc=%d)"
|
||||
% result.returncode
|
||||
)
|
||||
if not os.path.exists(linker_cmd):
|
||||
raise RuntimeError(
|
||||
"nRF54L15 linker fix: GCC returned 0 but linker.cmd was not created at %s"
|
||||
% linker_cmd
|
||||
)
|
||||
print("==> linker.cmd generated successfully")
|
||||
|
||||
# Use PIOMAINPROG (set by ZephyrBuildProgram) to get the exact SCons node
|
||||
prog = env.get("PIOMAINPROG")
|
||||
if prog:
|
||||
env.AddPreAction(prog, _generate_linker_cmd)
|
||||
else:
|
||||
print(
|
||||
"[nrf54l15_linker] WARNING: PIOMAINPROG not set, falling back to $PROG_PATH"
|
||||
)
|
||||
env.AddPreAction(env.subst("$PROG_PATH"), _generate_linker_cmd)
|
||||
@@ -33,10 +33,3 @@ tests/reproducers/
|
||||
# UI-tier camera captures + per-test transcripts. Regenerated every run;
|
||||
# left on disk for human review between runs.
|
||||
tests/ui_captures/
|
||||
|
||||
# Web stack (FleetSuite): node deps + the built SPA Vite emits into the
|
||||
# package (regenerated via `cd web-ui && npm run build`). The SQLite registry
|
||||
# lives under .mtlog/ (already ignored above).
|
||||
web-ui/node_modules/
|
||||
web-ui/dist/
|
||||
src/meshtastic_mcp/web/static/
|
||||
|
||||
+17
-37
@@ -174,7 +174,7 @@ rather than auto-`sudo`'ing mid-run.
|
||||
| `MESHTASTIC_NRFUTIL_BIN` | `$PATH` | Override nrfutil |
|
||||
| `MESHTASTIC_PICOTOOL_BIN` | `$PATH` | Override picotool |
|
||||
| `MESHTASTIC_MCP_SEED` | `mcp-<user>-<host>` | PSK seed for test-harness session (CI override) |
|
||||
| `MESHTASTIC_MCP_FLASH_LOG` | `<mcp-server>/tests/flash.log` | Tee target for pio/esptool/nrfutil subprocess output (web UI tails it) |
|
||||
| `MESHTASTIC_MCP_FLASH_LOG` | `<mcp-server>/tests/flash.log` | Tee target for pio/esptool/nrfutil subprocess output (TUI tails it) |
|
||||
| `MESHTASTIC_MCP_TCP_HOST` | unset | `host` or `host:port` of a `meshtasticd` daemon to surface as a TCP device (see "TCP / native-host nodes" below) |
|
||||
|
||||
## TCP / native-host nodes
|
||||
@@ -333,51 +333,31 @@ captures just become 1×1 black PNGs.
|
||||
**Meshtastic debug** section attached on failure with a 200-line firmware
|
||||
log tail + device-state dump. Open this first on failures.
|
||||
- `junit.xml` — CI-parseable.
|
||||
- `reportlog.jsonl` — `pytest-reportlog` event stream; consumed by the web UI.
|
||||
- `reportlog.jsonl` — `pytest-reportlog` event stream; consumed by the TUI.
|
||||
- `fwlog.jsonl` — firmware log mirror (`meshtastic.log.line` pubsub → JSONL).
|
||||
- `flash.log` — tee of all pio / esptool / nrfutil / picotool subprocess
|
||||
output during the run (driven by `MESHTASTIC_MCP_FLASH_LOG`).
|
||||
|
||||
### Web UI (FleetSuite)
|
||||
|
||||
A Vue 3 + Tailwind desktop web app replaces the old Textual TUI. It serves a
|
||||
FastAPI backend (REST + WebSocket) that reuses the library modules and a
|
||||
SQLite registry of devices and cameras. See [web-ui/README.md](web-ui/README.md).
|
||||
|
||||
**One command** (bootstraps the venv, web deps, npm packages, and SPA build on
|
||||
first run, then launches):
|
||||
### Live TUI
|
||||
|
||||
```bash
|
||||
./scripts/fleetsuite.sh # open the native desktop window
|
||||
./scripts/fleetsuite.sh --browser # serve only → http://127.0.0.1:8765
|
||||
./scripts/fleetsuite.sh --dev # backend + Vite dev server with hot-reload
|
||||
./scripts/fleetsuite.sh --rebuild # force a fresh SPA build first
|
||||
.venv/bin/meshtastic-mcp-test-tui
|
||||
.venv/bin/meshtastic-mcp-test-tui tests/mesh # pytest args pass through
|
||||
```
|
||||
|
||||
In **VS Code**: Run Task → **FleetSuite** (or _FleetSuite: Dev (HMR)_). Or do it
|
||||
by hand:
|
||||
Textual-based wrapper over `run-tests.sh` with a live test tree, tier
|
||||
counters, pytest output pane, firmware-log pane, and a device-status strip.
|
||||
Key bindings: `r` re-run focused, `f` filter, `d` failure detail, `g` open
|
||||
`report.html`, `x` export reproducer bundle, `l` cycle fw-log filter, `q`
|
||||
quit (SIGINT → SIGTERM → SIGKILL escalation).
|
||||
|
||||
```bash
|
||||
.venv/bin/pip install -e '.[web]' # FastAPI + uvicorn + aiosqlite + pywebview
|
||||
(cd web-ui && npm install && npm run build) # build the SPA into web/static
|
||||
.venv/bin/meshtastic-mcp-web # opens a native pywebview window
|
||||
.venv/bin/meshtastic-mcp-web --browser # serve only → http://127.0.0.1:8765
|
||||
```
|
||||
|
||||
Two views: **Fleet** (per-device cards keyed by USB serial that follow a device
|
||||
across changing ports, with live serial logs, packet/telemetry, test history,
|
||||
full device control, and an assignable live USB camera feed) and **Test Suite**
|
||||
(run/stop `run-tests.sh`, live tier counters + test tree, pytest/flash/firmware
|
||||
log panes, run history). The header shows the firmware branch/SHA/dirty live.
|
||||
|
||||
**Native nodes (Docker).** The Fleet view can run Linux-native `meshtasticd`
|
||||
nodes in Docker (no radio, sim mode) and manage them as TCP devices — run /
|
||||
stop / restart / remove, live `docker logs`, config/send-text/inject-NodeDB over
|
||||
`tcp://`. By default it builds the checkout's own image from `Dockerfile`; set
|
||||
`MESHTASTIC_NATIVE_IMAGE` (e.g. `meshtastic/meshtasticd:latest`) to use a
|
||||
prebuilt image instead.
|
||||
|
||||
For development with HMR, run `scripts/web-dev.sh` (backend + Vite together).
|
||||
Set `MESHTASTIC_UI_TUI_CAMERA=1` to mount a bottom-of-screen **UI camera**
|
||||
panel. Left side: the latest capture PNG rendered as Unicode half-blocks
|
||||
(via `rich-pixels`, works in any terminal — no kitty/sixel required).
|
||||
Right side: live transcript tail ("step 3 — frame 4/8 name=nodelist_nodes
|
||||
— OCR: Nodes 2/2") so you can see every event-injection and its result
|
||||
as each UI test runs. Requires the `[ui]` extras for image rendering; the
|
||||
transcript alone works without them.
|
||||
|
||||
### Slash commands
|
||||
|
||||
|
||||
+13
-14
@@ -17,16 +17,11 @@ test = [
|
||||
"pytest-timeout>=2.3",
|
||||
"coverage[toml]>=7",
|
||||
"pyyaml>=6",
|
||||
]
|
||||
# Web stack (FastAPI backend + pywebview desktop window) that replaces the
|
||||
# Textual TUI. Serves the built Vue SPA in `web/static`. Camera streaming
|
||||
# reuses the `[ui]` extra (opencv-python-headless).
|
||||
web = [
|
||||
"fastapi>=0.110",
|
||||
"uvicorn[standard]>=0.27",
|
||||
"aiosqlite>=0.19",
|
||||
"pywebview>=5.0",
|
||||
"requests>=2.31",
|
||||
# textual is required by the `meshtastic-mcp-test-tui` script (see
|
||||
# `src/meshtastic_mcp/cli/test_tui.py`). Bundled into `test` rather than a
|
||||
# separate `[tui]` extra because v1 expects test operators are the only
|
||||
# consumers; revisit if install cost pushes back.
|
||||
"textual>=0.50",
|
||||
]
|
||||
# UI test tier + `capture_screen` MCP tool. Optional because the ML OCR
|
||||
# model alone is ~100 MB and camera hardware is user-supplied.
|
||||
@@ -37,15 +32,19 @@ ui = [
|
||||
"numpy>=1.26",
|
||||
"easyocr>=1.7",
|
||||
"Pillow>=10.0",
|
||||
# Renders the latest camera capture as Unicode half-blocks in the TUI
|
||||
# (MESHTASTIC_UI_TUI_CAMERA=1). Terminal-agnostic — no kitty / sixel
|
||||
# dependency. Pure Python, tiny.
|
||||
"rich-pixels>=3.0",
|
||||
]
|
||||
ui-min = ["opencv-python-headless>=4.9", "numpy>=1.26"]
|
||||
|
||||
[project.scripts]
|
||||
meshtastic-mcp = "meshtastic_mcp.__main__:main"
|
||||
# Web stack replacing the Textual TUI: a FastAPI backend (REST + WebSocket)
|
||||
# that reuses the library modules and serves the Vue SPA, opened in a
|
||||
# pywebview window (`--browser` to serve only).
|
||||
meshtastic-mcp-web = "meshtastic_mcp.web.__main__:main"
|
||||
# Live TUI wrapping run-tests.sh — shells out to the same script the plain
|
||||
# CLI uses, tails pytest-reportlog for per-test state, and polls the device
|
||||
# list at startup + post-run (port lock forces it to stay idle during the run).
|
||||
meshtastic-mcp-test-tui = "meshtastic_mcp.cli.test_tui:main"
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
|
||||
@@ -191,12 +191,10 @@ echo
|
||||
# PASS/FAIL — every hardware test would SKIP with "role not present". We
|
||||
# narrow to tests/unit explicitly so the summary reads as "no hardware,
|
||||
# unit suite only" instead of "big skip count looks suspicious".
|
||||
# Keep terminal output condensed (`-q -r fE`) so skip-heavy runs do not print
|
||||
# each skipped test in full; skip counts still appear in pytest's summary.
|
||||
if [[ -z $DETECTED && $# -eq 0 ]]; then
|
||||
echo "[pre-flight] no supported devices detected; running unit tier only."
|
||||
echo
|
||||
exec "$VENV_PY" -m pytest tests/unit -q -r fE --report-log=tests/reportlog.jsonl
|
||||
exec "$VENV_PY" -m pytest tests/unit -v --report-log=tests/reportlog.jsonl
|
||||
fi
|
||||
|
||||
# Default pytest args when the user passed none. Power users can invoke
|
||||
@@ -212,13 +210,11 @@ fi
|
||||
# skipping half the hardware tests with "not baked with session profile"
|
||||
# errors. Power users who know their hardware is current and want to shave
|
||||
# those seconds can pass `--assume-baked` explicitly.
|
||||
# Defaults also use condensed reporting (`-q -r fE`) to avoid listing every
|
||||
# skipped test verbatim while still surfacing failures/errors and summary data.
|
||||
if [[ $# -eq 0 ]]; then
|
||||
set -- tests/ \
|
||||
--html=tests/report.html --self-contained-html \
|
||||
--junitxml=tests/junit.xml \
|
||||
-q -r fE --tb=short
|
||||
-v --tb=short
|
||||
fi
|
||||
|
||||
# UI tier requires opencv-python-headless (and ideally easyocr). If it's
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Single entrypoint for FleetSuite — the web UI for the Meshtastic test harness.
|
||||
# Ensures deps, builds the SPA, and launches the app. One command, from anywhere.
|
||||
#
|
||||
# ./scripts/fleetsuite.sh # build SPA if needed, open the desktop window
|
||||
# ./scripts/fleetsuite.sh --browser # serve only → http://127.0.0.1:8765
|
||||
# ./scripts/fleetsuite.sh --dev # backend + Vite dev server with hot-reload
|
||||
# ./scripts/fleetsuite.sh --rebuild # force a fresh SPA build first
|
||||
#
|
||||
# First run bootstraps the venv + web deps + npm packages automatically.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
DEV=0
|
||||
BROWSER=0
|
||||
REBUILD=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--dev) DEV=1 ;;
|
||||
--browser) BROWSER=1 ;;
|
||||
--rebuild) REBUILD=1 ;;
|
||||
-h | --help)
|
||||
sed -n '2,11p' "${BASH_SOURCE[0]}"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "unknown arg: $arg (try --help)" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
PY="$ROOT/.venv/bin/python"
|
||||
STATIC="$ROOT/src/meshtastic_mcp/web/static/index.html"
|
||||
|
||||
note() { printf '\033[36m[fleetsuite]\033[0m %s\n' "$*"; }
|
||||
|
||||
# 1. Python venv + web extra ------------------------------------------------
|
||||
if [[ ! -x $PY ]]; then
|
||||
note "creating venv (.venv)…"
|
||||
python3 -m venv "$ROOT/.venv"
|
||||
fi
|
||||
if ! "$PY" -c 'import fastapi, aiosqlite, uvicorn, webview' >/dev/null 2>&1; then
|
||||
note "installing the [web] extra…"
|
||||
"$PY" -m pip install --quiet --upgrade pip
|
||||
"$PY" -m pip install --quiet -e "$ROOT[web]"
|
||||
fi
|
||||
|
||||
# 2. Dev mode: backend + Vite with HMR --------------------------------------
|
||||
if [[ $DEV == 1 ]]; then
|
||||
note "dev mode (backend :8765 + Vite HMR)"
|
||||
exec "$ROOT/scripts/web-dev.sh"
|
||||
fi
|
||||
|
||||
# 3. Frontend deps + production build ----------------------------------------
|
||||
if ! command -v npm >/dev/null 2>&1; then
|
||||
echo "npm not found — install Node.js (https://nodejs.org) to build the UI." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -d "$ROOT/web-ui/node_modules" ]]; then
|
||||
note "installing web-ui npm packages…"
|
||||
(cd "$ROOT/web-ui" && npm install)
|
||||
fi
|
||||
if [[ $REBUILD == 1 || ! -f $STATIC ]]; then
|
||||
note "building the SPA…"
|
||||
(cd "$ROOT/web-ui" && npm run build)
|
||||
fi
|
||||
|
||||
# 4. Launch ------------------------------------------------------------------
|
||||
if [[ $BROWSER == 1 ]]; then
|
||||
note "serving at http://127.0.0.1:8765 (Ctrl-C to stop)"
|
||||
exec "$ROOT/.venv/bin/meshtastic-mcp-web" --browser
|
||||
fi
|
||||
note "opening FleetSuite window…"
|
||||
exec "$ROOT/.venv/bin/meshtastic-mcp-web"
|
||||
@@ -1,32 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run the FleetSuite backend (uvicorn, :8765) and the Vite dev server together,
|
||||
# with HMR. The Vite server proxies /api and /ws to the backend. Ctrl-C stops
|
||||
# both. For a single-process production run instead, build the SPA once
|
||||
# (`cd web-ui && npm run build`) and launch `meshtastic-mcp-web`.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
PY="${ROOT}/.venv/bin/python"
|
||||
if [[ ! -x $PY ]]; then
|
||||
echo "No venv at .venv — run: python3 -m venv .venv && .venv/bin/pip install -e '.[web,test]'" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cleanup() {
|
||||
trap - INT TERM
|
||||
[[ -n ${BACK_PID-} ]] && kill "$BACK_PID" 2>/dev/null || true
|
||||
[[ -n ${UI_PID-} ]] && kill "$UI_PID" 2>/dev/null || true
|
||||
}
|
||||
trap cleanup INT TERM EXIT
|
||||
|
||||
echo "[web-dev] starting backend on http://127.0.0.1:8765 …"
|
||||
"$PY" -m uvicorn meshtastic_mcp.web.app:create_app --factory --port 8765 --reload &
|
||||
BACK_PID=$!
|
||||
|
||||
echo "[web-dev] starting Vite dev server …"
|
||||
(cd web-ui && npm run dev) &
|
||||
UI_PID=$!
|
||||
|
||||
wait
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,382 +0,0 @@
|
||||
"""Fake NodeDB fixture push — Portduino file copy + hardware XModem upload.
|
||||
|
||||
The fixture pipeline is two-stage:
|
||||
1. `bin/gen-fake-nodedb-seed.py` produces a deterministic JSONL describing N
|
||||
fake-but-realistic peers. Committed under `test/fixtures/nodedb/`.
|
||||
2. `bin/seed-json-to-proto.py` compiles JSONL → binary v25 NodeDatabase
|
||||
protobuf with fresh wall-clock timestamps.
|
||||
|
||||
This module exposes `push_fake_nodedb(...)`, the MCP tool that:
|
||||
- target="portduino": compiles the JSONL into the device's prefs dir on
|
||||
the local filesystem (`~/.portduino/<config>/prefs/nodes.proto`).
|
||||
- target="hardware": compiles to a temp file, then streams it over the
|
||||
XModem protocol (via the meshtastic SerialInterface/BLEInterface +
|
||||
`meshtastic.xmodempacket` pubsub topic) to `/prefs/nodes.proto` on the
|
||||
device. Triggers a reboot so the firmware loads the new state on next
|
||||
boot.
|
||||
|
||||
XModem wire details (mirrors firmware impl at src/xmodem.cpp:115-260):
|
||||
* 128-byte chunks; final chunk padded to 128 B with 0x1A (SUB) bytes.
|
||||
* CRC16-CCITT (poly 0x1021, init 0x0000).
|
||||
* SOH/seq=0 carries the destination filename in `buffer.bytes`. ACK if
|
||||
`FSCom.open(filename, FILE_O_WRITE)` succeeds; NAK otherwise.
|
||||
* SOH/seq≥1 carries a 128-byte chunk. ACK = advance; NAK = retransmit.
|
||||
* EOT after the last chunk flushes + closes the file on-device.
|
||||
|
||||
Hardware push requires `confirm=True` (mirrors factory_reset / erase_and_flash
|
||||
in the .github/copilot-instructions.md "never do these without asking" list).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import pathlib
|
||||
import queue
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from typing import Any, Literal
|
||||
|
||||
from .connection import connect, is_tcp_port
|
||||
|
||||
# Resolve repo root so the tool works regardless of mcp-server cwd.
|
||||
_REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]
|
||||
_SEED_DIR = _REPO_ROOT / "test" / "fixtures" / "nodedb"
|
||||
_COMPILE_SCRIPT = _REPO_ROOT / "bin" / "seed-json-to-proto.py"
|
||||
|
||||
_DEFAULT_NODES_FILENAME = "/prefs/nodes.proto"
|
||||
_XMODEM_CHUNK = 128
|
||||
_XMODEM_SUB = 0x1A
|
||||
_ACK_TIMEOUT_INIT_S = 5.0
|
||||
_ACK_TIMEOUT_CHUNK_S = 2.0
|
||||
_MAX_CHUNK_RETRIES = 5
|
||||
|
||||
_VALID_SIZES = (250, 500, 1000, 2000)
|
||||
|
||||
|
||||
class FixtureError(RuntimeError):
|
||||
"""Raised for any fixture-push failure (compile, transport, ack timeout, …)."""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CRC16-CCITT (poly 0x1021, init 0x0000). Matches the firmware's `crc16_ccitt`.
|
||||
# Hand-rolled to avoid the optional `crcmod` dep.
|
||||
# ---------------------------------------------------------------------------
|
||||
def _crc16_ccitt(data: bytes, *, init: int = 0x0000) -> int:
|
||||
crc = init
|
||||
for b in data:
|
||||
crc ^= b << 8
|
||||
for _ in range(8):
|
||||
if crc & 0x8000:
|
||||
crc = ((crc << 1) ^ 0x1021) & 0xFFFF
|
||||
else:
|
||||
crc = (crc << 1) & 0xFFFF
|
||||
return crc
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Compile step — shells out to bin/seed-json-to-proto.py so the MCP module
|
||||
# doesn't have to duplicate the proto-encoding logic.
|
||||
# ---------------------------------------------------------------------------
|
||||
def _compile_proto(jsonl_path: pathlib.Path, out_path: pathlib.Path) -> None:
|
||||
if not _COMPILE_SCRIPT.is_file():
|
||||
raise FixtureError(f"compile script missing at {_COMPILE_SCRIPT}")
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(_COMPILE_SCRIPT),
|
||||
"--in",
|
||||
str(jsonl_path),
|
||||
"--out",
|
||||
str(out_path),
|
||||
]
|
||||
try:
|
||||
subprocess.run(cmd, check=True, capture_output=True, text=True)
|
||||
except subprocess.CalledProcessError as exc:
|
||||
raise FixtureError(
|
||||
f"seed-json-to-proto.py failed (exit {exc.returncode}):\n"
|
||||
f" stdout: {exc.stdout}\n stderr: {exc.stderr}"
|
||||
) from exc
|
||||
|
||||
|
||||
def _resolve_seed_jsonl(size: int, custom: str | None) -> pathlib.Path:
|
||||
if custom is not None:
|
||||
p = pathlib.Path(custom).expanduser().resolve()
|
||||
if not p.is_file():
|
||||
raise FixtureError(f"custom_seed_jsonl not found: {p}")
|
||||
return p
|
||||
p = _SEED_DIR / f"seed_v25_{size:04d}.jsonl"
|
||||
if not p.is_file():
|
||||
raise FixtureError(
|
||||
f"missing committed seed at {p}. "
|
||||
f"Run `./bin/regen-fake-nodedbs.sh` to generate it."
|
||||
)
|
||||
return p
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Portduino push — file copy into ~/.portduino/<config>/prefs/
|
||||
# ---------------------------------------------------------------------------
|
||||
def _portduino_prefs_dir(config_name: str) -> pathlib.Path:
|
||||
home = pathlib.Path.home()
|
||||
return home / ".portduino" / config_name / "prefs"
|
||||
|
||||
|
||||
def _push_portduino(
|
||||
size: int,
|
||||
jsonl: pathlib.Path,
|
||||
portduino_config: str,
|
||||
backup_existing: bool,
|
||||
) -> dict[str, Any]:
|
||||
prefs = _portduino_prefs_dir(portduino_config)
|
||||
prefs.mkdir(parents=True, exist_ok=True)
|
||||
target = prefs / "nodes.proto"
|
||||
backed_up_to: str | None = None
|
||||
if backup_existing and target.is_file():
|
||||
ts = int(time.time())
|
||||
backup = prefs / f"nodes.proto.bak.{ts}"
|
||||
shutil.move(str(target), str(backup))
|
||||
backed_up_to = str(backup)
|
||||
_compile_proto(jsonl, target)
|
||||
raw = target.read_bytes()
|
||||
return {
|
||||
"transport": "portduino",
|
||||
"path": str(target),
|
||||
"bytes": len(raw),
|
||||
"sha256": hashlib.sha256(raw).hexdigest(),
|
||||
"jsonl_source": str(jsonl),
|
||||
"backed_up_to": backed_up_to,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hardware push — XModem over BLE/serial via the meshtastic Python interface.
|
||||
# ---------------------------------------------------------------------------
|
||||
@dataclasses.dataclass
|
||||
class _AckEvent:
|
||||
control: int
|
||||
seq: int
|
||||
|
||||
|
||||
def _wait_for_response(q: "queue.Queue[_AckEvent]", timeout_s: float) -> _AckEvent:
|
||||
try:
|
||||
return q.get(timeout=timeout_s)
|
||||
except queue.Empty as exc:
|
||||
raise FixtureError(
|
||||
f"XModem response timeout after {timeout_s:.1f}s — device not responding"
|
||||
) from exc
|
||||
|
||||
|
||||
def _push_hardware(
|
||||
size: int,
|
||||
jsonl: pathlib.Path,
|
||||
port: str | None,
|
||||
reboot_after: bool,
|
||||
) -> dict[str, Any]:
|
||||
# Lazy imports so the module loads even when the meshtastic deps aren't
|
||||
# available (e.g. CI in a Python env without the package installed).
|
||||
try:
|
||||
from meshtastic.protobuf import mesh_pb2, xmodem_pb2
|
||||
from pubsub import pub
|
||||
except ImportError as exc: # pragma: no cover — dep missing
|
||||
raise FixtureError(
|
||||
f"hardware push requires the meshtastic + pypubsub packages: {exc}"
|
||||
) from exc
|
||||
|
||||
if is_tcp_port(port):
|
||||
raise FixtureError(
|
||||
"hardware push over TCP/portduino is not supported — use "
|
||||
"target='portduino' to drop the fixture directly into the prefs dir."
|
||||
)
|
||||
|
||||
# Compile the fixture to a temp file with fresh timestamps.
|
||||
with tempfile.NamedTemporaryFile(suffix=".proto", delete=False) as tf:
|
||||
proto_path = pathlib.Path(tf.name)
|
||||
try:
|
||||
_compile_proto(jsonl, proto_path)
|
||||
payload = proto_path.read_bytes()
|
||||
finally:
|
||||
proto_path.unlink(missing_ok=True)
|
||||
|
||||
sha256 = hashlib.sha256(payload).hexdigest()
|
||||
total_bytes = len(payload)
|
||||
|
||||
# Subscribe to XModem responses BEFORE we open the interface, so we don't
|
||||
# race the first ACK that arrives during the SOH/seq=0 handshake.
|
||||
#
|
||||
# NB: the signature MUST declare every kwarg pypubsub will see for this
|
||||
# topic, or pubsub locks the topic spec to a smaller set (whichever
|
||||
# subscribe arrives first) and then *rejects* the meshtastic library's
|
||||
# publish call with `SenderUnknownMsgDataError: unknown ... interface`.
|
||||
# The meshtastic lib publishes both `packet=` and `interface=`
|
||||
# (mesh_interface.py:1389-1395), so both must appear here.
|
||||
response_q: "queue.Queue[_AckEvent]" = queue.Queue()
|
||||
|
||||
def _on_xmodem(packet: Any = None, interface: Any = None, **_kw: Any) -> None:
|
||||
if packet is None:
|
||||
return
|
||||
response_q.put(_AckEvent(control=int(packet.control), seq=int(packet.seq)))
|
||||
|
||||
pub.subscribe(_on_xmodem, "meshtastic.xmodempacket")
|
||||
|
||||
chunks_sent = 0
|
||||
retried = 0
|
||||
rebooted = False
|
||||
|
||||
XMC = xmodem_pb2.XModem.Control
|
||||
try:
|
||||
with connect(port=port) as iface:
|
||||
# 1) Send the filename (SOH, seq=0).
|
||||
init_pkt = xmodem_pb2.XModem(
|
||||
control=XMC.Value("SOH"),
|
||||
seq=0,
|
||||
buffer=_DEFAULT_NODES_FILENAME.encode("utf-8"),
|
||||
)
|
||||
iface._sendToRadio(mesh_pb2.ToRadio(xmodemPacket=init_pkt))
|
||||
ack = _wait_for_response(response_q, _ACK_TIMEOUT_INIT_S)
|
||||
if ack.control != XMC.Value("ACK"):
|
||||
raise FixtureError(
|
||||
f"device refused filename {_DEFAULT_NODES_FILENAME!r} "
|
||||
f"(got control={ack.control}, expected ACK). "
|
||||
f"Filesystem full or permissions issue?"
|
||||
)
|
||||
|
||||
# 2) Stream the payload in 128 B chunks.
|
||||
for offset in range(0, total_bytes, _XMODEM_CHUNK):
|
||||
chunk = payload[offset : offset + _XMODEM_CHUNK]
|
||||
if len(chunk) < _XMODEM_CHUNK:
|
||||
# Pad final chunk to 128 B with SUB. The trailing 0x1A bytes
|
||||
# become part of the file on-device, but nanopb ignores
|
||||
# bytes past the end of the top-level message.
|
||||
chunk = chunk + bytes([_XMODEM_SUB] * (_XMODEM_CHUNK - len(chunk)))
|
||||
seq = ((offset // _XMODEM_CHUNK) + 1) % 256
|
||||
# Retry loop on NAK / timeout.
|
||||
attempts = 0
|
||||
while True:
|
||||
pkt = xmodem_pb2.XModem(
|
||||
control=XMC.Value("SOH"),
|
||||
seq=seq,
|
||||
buffer=chunk,
|
||||
crc16=_crc16_ccitt(chunk),
|
||||
)
|
||||
iface._sendToRadio(mesh_pb2.ToRadio(xmodemPacket=pkt))
|
||||
ack = _wait_for_response(response_q, _ACK_TIMEOUT_CHUNK_S)
|
||||
if ack.control == XMC.Value("ACK"):
|
||||
chunks_sent += 1
|
||||
break
|
||||
if ack.control == XMC.Value("NAK"):
|
||||
attempts += 1
|
||||
retried += 1
|
||||
if attempts >= _MAX_CHUNK_RETRIES:
|
||||
# Abort: send CAN so the firmware removes the half-
|
||||
# written file via FSCom.remove(filename).
|
||||
iface._sendToRadio(
|
||||
mesh_pb2.ToRadio(
|
||||
xmodemPacket=xmodem_pb2.XModem(
|
||||
control=XMC.Value("CAN")
|
||||
)
|
||||
)
|
||||
)
|
||||
raise FixtureError(
|
||||
f"chunk seq={seq} NAK'd {attempts} times; "
|
||||
f"aborted transfer (file removed on-device)."
|
||||
)
|
||||
continue # retry the same chunk
|
||||
raise FixtureError(
|
||||
f"unexpected XModem control={ack.control} on seq={seq}"
|
||||
)
|
||||
|
||||
# 3) Tell the device we're done.
|
||||
iface._sendToRadio(
|
||||
mesh_pb2.ToRadio(
|
||||
xmodemPacket=xmodem_pb2.XModem(control=XMC.Value("EOT"))
|
||||
)
|
||||
)
|
||||
ack = _wait_for_response(response_q, _ACK_TIMEOUT_CHUNK_S)
|
||||
if ack.control != XMC.Value("ACK"):
|
||||
raise FixtureError(f"EOT not ACKed (got control={ack.control})")
|
||||
|
||||
# 4) Reboot so loadFromDisk picks up the new file.
|
||||
if reboot_after:
|
||||
iface.localNode.reboot(secs=1)
|
||||
rebooted = True
|
||||
finally:
|
||||
try:
|
||||
pub.unsubscribe(_on_xmodem, "meshtastic.xmodempacket")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {
|
||||
"transport": "hardware",
|
||||
"port": port,
|
||||
"filename_on_device": _DEFAULT_NODES_FILENAME,
|
||||
"bytes": total_bytes,
|
||||
"chunks_sent": chunks_sent,
|
||||
"retried": retried,
|
||||
"sha256": sha256,
|
||||
"jsonl_source": str(jsonl),
|
||||
"rebooted": rebooted,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public entry point — registered as an MCP tool in server.py.
|
||||
# ---------------------------------------------------------------------------
|
||||
def push_fake_nodedb(
|
||||
size: int,
|
||||
target: Literal["portduino", "hardware"] = "portduino",
|
||||
*,
|
||||
port: str | None = None,
|
||||
portduino_config: str = "default",
|
||||
backup_existing: bool = True,
|
||||
confirm: bool = False,
|
||||
reboot_after: bool = True,
|
||||
custom_seed_jsonl: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Compile a fresh-timestamp NodeDatabase fixture and push it to a device.
|
||||
|
||||
Args:
|
||||
size: 250, 500, 1000, or 2000 — selects which committed seed JSONL to use.
|
||||
target: "portduino" (file copy to ~/.portduino/<config>/prefs/) or
|
||||
"hardware" (XModem upload to /prefs/nodes.proto + reboot).
|
||||
port: required for target="hardware". Serial path (e.g. /dev/cu.usbmodemXXXX)
|
||||
or BLE identifier. TCP endpoints are rejected — use target="portduino"
|
||||
instead.
|
||||
portduino_config: which Portduino instance dir under ~/.portduino/. Default "default".
|
||||
backup_existing: portduino only. Move nodes.proto -> nodes.proto.bak.<ts>
|
||||
if present, so you can roll back.
|
||||
confirm: required True for target="hardware" (writes flash + reboots).
|
||||
reboot_after: hardware only. If True, send a 1-second reboot after the
|
||||
final ACK so loadFromDisk picks up the new file at next boot.
|
||||
custom_seed_jsonl: override the committed JSONL. Use to push a hand-edited
|
||||
test scenario.
|
||||
|
||||
Returns:
|
||||
dict with transport, bytes, sha256, etc. — depends on target.
|
||||
|
||||
"""
|
||||
if size not in _VALID_SIZES:
|
||||
raise FixtureError(
|
||||
f"size must be one of {_VALID_SIZES}; got {size!r}. "
|
||||
f"Add a new committed seed if you need a different cardinality."
|
||||
)
|
||||
|
||||
jsonl = _resolve_seed_jsonl(size, custom_seed_jsonl)
|
||||
|
||||
if target == "portduino":
|
||||
return _push_portduino(size, jsonl, portduino_config, backup_existing)
|
||||
|
||||
if target == "hardware":
|
||||
if not confirm:
|
||||
raise FixtureError(
|
||||
"hardware push writes flash and triggers a reboot — pass confirm=True."
|
||||
)
|
||||
if not port:
|
||||
raise FixtureError(
|
||||
"target='hardware' requires a port (e.g. /dev/cu.usbmodemXXXX)."
|
||||
)
|
||||
return _push_hardware(size, jsonl, port, reboot_after)
|
||||
|
||||
raise FixtureError(f"unknown target {target!r}; expected 'portduino' or 'hardware'")
|
||||
@@ -15,7 +15,6 @@ from . import (
|
||||
admin,
|
||||
boards,
|
||||
devices,
|
||||
fixtures,
|
||||
flash,
|
||||
hw_tools,
|
||||
info,
|
||||
@@ -964,45 +963,3 @@ def recorder_export(
|
||||
dest_dir=dest_dir,
|
||||
streams=streams,
|
||||
)
|
||||
|
||||
|
||||
# ---------- Fixture / test-data push --------------------------------------
|
||||
|
||||
|
||||
@app.tool()
|
||||
def push_fake_nodedb(
|
||||
size: int,
|
||||
target: str = "portduino",
|
||||
port: str | None = None,
|
||||
portduino_config: str = "default",
|
||||
backup_existing: bool = True,
|
||||
confirm: bool = False,
|
||||
reboot_after: bool = True,
|
||||
custom_seed_jsonl: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Push a fake-NodeDB v25 fixture (250/500/1000/2000 nodes) onto a device.
|
||||
|
||||
Two transports:
|
||||
target="portduino" — file copy to ~/.portduino/<portduino_config>/prefs/nodes.proto.
|
||||
Fast, no device connection needed.
|
||||
target="hardware" — XModem upload over serial/BLE to /prefs/nodes.proto.
|
||||
Requires `port` + `confirm=True`. Triggers a reboot
|
||||
so loadFromDisk picks up the new file at next boot.
|
||||
|
||||
Compiles a fresh-timestamp proto from the committed JSONL seed under
|
||||
test/fixtures/nodedb/seed_v25_<N>.jsonl each invocation, so the loaded
|
||||
NodeDB always looks "recent" to the connecting phone. Structural data
|
||||
(names, IDs, positions, telemetries) is deterministic per the seed.
|
||||
|
||||
Override the JSONL via `custom_seed_jsonl` to push a hand-edited scenario.
|
||||
"""
|
||||
return fixtures.push_fake_nodedb(
|
||||
size=size,
|
||||
target=target, # type: ignore[arg-type]
|
||||
port=port,
|
||||
portduino_config=portduino_config,
|
||||
backup_existing=backup_existing,
|
||||
confirm=confirm,
|
||||
reboot_after=reboot_after,
|
||||
custom_seed_jsonl=custom_seed_jsonl,
|
||||
)
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
"""FleetSuite — the FastAPI + WebSocket backend that serves the Vue SPA and
|
||||
drives the Meshtastic test harness. Replaces the old Textual TUI.
|
||||
|
||||
Layout:
|
||||
db/ — aiosqlite registry (devices, cameras, flash, runs, builds, settings)
|
||||
services/ — identity reconciliation, control gating, firmware builds, the
|
||||
pytest runner, the Datadog forwarder
|
||||
ws/ — the single broadcast hub backing ``/ws``
|
||||
app.py — ``create_app()`` factory (REST + ``/ws``, serves ``web/static``)
|
||||
__main__.py — ``main()``: serve + open a pywebview window (``--browser`` to skip it)
|
||||
"""
|
||||
@@ -1,79 +0,0 @@
|
||||
"""FleetSuite entrypoint (the ``meshtastic-mcp-web`` console script).
|
||||
|
||||
Default: serve the API + built SPA on 127.0.0.1:8765 and open a pywebview
|
||||
desktop window pointed at it. ``--browser`` serves only (no window) so you can
|
||||
open it in any browser — also the mode used in headless/CI and by agents.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
|
||||
import uvicorn
|
||||
|
||||
HOST = "127.0.0.1"
|
||||
PORT = 8765
|
||||
|
||||
|
||||
def _serve(server: uvicorn.Server) -> None:
|
||||
server.run()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(prog="meshtastic-mcp-web", description=__doc__)
|
||||
parser.add_argument(
|
||||
"--browser",
|
||||
action="store_true",
|
||||
help="serve only (no desktop window) — open http://127.0.0.1:8765 yourself",
|
||||
)
|
||||
parser.add_argument("--host", default=HOST)
|
||||
parser.add_argument("--port", type=int, default=PORT)
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s"
|
||||
)
|
||||
|
||||
config = uvicorn.Config(
|
||||
"meshtastic_mcp.web.app:create_app",
|
||||
factory=True,
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
log_level="info",
|
||||
)
|
||||
server = uvicorn.Server(config)
|
||||
|
||||
if args.browser:
|
||||
server.run()
|
||||
return
|
||||
|
||||
# Desktop window: run uvicorn on a background thread, open pywebview on the
|
||||
# main thread (some platforms require the GUI loop to own the main thread).
|
||||
try:
|
||||
import webview # type: ignore
|
||||
except Exception: # noqa: BLE001
|
||||
logging.warning("pywebview unavailable — falling back to --browser mode")
|
||||
server.run()
|
||||
return
|
||||
|
||||
thread = threading.Thread(target=_serve, args=(server,), daemon=True)
|
||||
thread.start()
|
||||
|
||||
# Wait for the server to bind before pointing the window at it.
|
||||
deadline = time.monotonic() + 15
|
||||
while not server.started and time.monotonic() < deadline:
|
||||
time.sleep(0.1)
|
||||
|
||||
webview.create_window(
|
||||
"FleetSuite", f"http://{args.host}:{args.port}", width=1400, height=900
|
||||
)
|
||||
webview.start()
|
||||
server.should_exit = True
|
||||
thread.join(timeout=5)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,611 +0,0 @@
|
||||
"""FastAPI application factory for FleetSuite.
|
||||
|
||||
``create_app()`` wires the registry, the broadcast hub, and the services
|
||||
together in a lifespan, mounts the REST API + the single ``/ws`` socket, and
|
||||
serves the built Vue SPA from ``web/static``. Blocking library calls (serial
|
||||
I/O, pio, git) are dispatched to a thread so the event loop stays responsive.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Body, FastAPI, HTTPException, Request, WebSocket
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
from meshtastic_mcp import (
|
||||
admin,
|
||||
boards,
|
||||
fixtures,
|
||||
flash as flash_lib,
|
||||
info as mt_info,
|
||||
log_query,
|
||||
)
|
||||
|
||||
from .db import repo_builds as rb
|
||||
from .db import repo_cameras as rc
|
||||
from .db import repo_devices as rd
|
||||
from .db import repo_flash as rf
|
||||
from .db import repo_runs as rr
|
||||
from .db.database import Database, default_db_path
|
||||
from .services import (
|
||||
builder,
|
||||
camera_stream,
|
||||
control,
|
||||
datadog,
|
||||
discovery,
|
||||
firmware,
|
||||
identity,
|
||||
native,
|
||||
power,
|
||||
serial_monitor,
|
||||
test_runner,
|
||||
)
|
||||
from .services.control import ControlBusy
|
||||
from .services.power import AmbiguousPort, NoPort
|
||||
from .ws.hub import Connection, Hub
|
||||
|
||||
log = logging.getLogger("meshtastic_mcp.web")
|
||||
|
||||
STATIC_DIR = Path(__file__).parent / "static"
|
||||
|
||||
|
||||
def _busy_guard(exc: ControlBusy) -> HTTPException:
|
||||
return HTTPException(status_code=409, detail=str(exc))
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
app = FastAPI(title="FleetSuite", version="0.1.0")
|
||||
|
||||
# --- lifespan: own the db + services for the process lifetime ----------
|
||||
@app.on_event("startup")
|
||||
async def _startup() -> None:
|
||||
db = await Database(default_db_path()).connect()
|
||||
hub = Hub()
|
||||
hub.bind_loop(asyncio.get_running_loop())
|
||||
|
||||
app.state.db = db
|
||||
app.state.hub = hub
|
||||
app.state.orch = builder.BuildOrchestrator(db, hub)
|
||||
app.state.runner = test_runner.TestRunner(db, hub)
|
||||
app.state.forwarder = datadog.DDForwarder(db, hub)
|
||||
await app.state.forwarder.reload()
|
||||
app.state.serialmon = serial_monitor.SerialMonitor(db, hub)
|
||||
# Discovery auto-enriches devices, suspending their serial monitor for
|
||||
# the connect — so it needs the monitor handle.
|
||||
app.state.discovery = discovery.DeviceDiscovery(
|
||||
db, hub, serialmon=app.state.serialmon
|
||||
)
|
||||
app.state.discovery.start()
|
||||
log.info("FleetSuite started — registry at %s", db.path)
|
||||
|
||||
@app.on_event("shutdown")
|
||||
async def _shutdown() -> None:
|
||||
disc = getattr(app.state, "discovery", None)
|
||||
if disc:
|
||||
await disc.stop()
|
||||
sm = getattr(app.state, "serialmon", None)
|
||||
if sm:
|
||||
await sm.shutdown()
|
||||
db = getattr(app.state, "db", None)
|
||||
if db:
|
||||
await db.close()
|
||||
|
||||
api = APIRouter(prefix="/api")
|
||||
_mount_devices(api)
|
||||
_mount_cameras(api)
|
||||
_mount_firmware(api)
|
||||
_mount_builds(api)
|
||||
_mount_datadog(api)
|
||||
_mount_tests(api)
|
||||
_mount_native(api)
|
||||
_mount_boards(api)
|
||||
_mount_hubs(api)
|
||||
app.include_router(api)
|
||||
_mount_ws(app)
|
||||
|
||||
@app.exception_handler(ControlBusy)
|
||||
async def _busy(_req: Request, exc: ControlBusy):
|
||||
return JSONResponse(status_code=409, content={"detail": str(exc)})
|
||||
|
||||
if STATIC_DIR.is_dir():
|
||||
app.mount("/", StaticFiles(directory=str(STATIC_DIR), html=True), name="spa")
|
||||
else:
|
||||
log.warning("no built SPA at %s — run `npm run build` in web-ui/", STATIC_DIR)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
# --- helpers ---------------------------------------------------------------
|
||||
async def _device_or_404(db: Database, serial: str) -> dict:
|
||||
row = await rd.get(db, serial)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail=f"unknown device: {serial}")
|
||||
return row
|
||||
|
||||
|
||||
def _gate_idle() -> None:
|
||||
try:
|
||||
control._ensure_idle()
|
||||
except ControlBusy as exc:
|
||||
raise _busy_guard(exc)
|
||||
|
||||
|
||||
async def _port_action(request: Request, serial: str, fn, *args):
|
||||
"""Run a blocking port-bound library call, suspending any live serial
|
||||
monitor for the device so the USB port is free, then resuming it."""
|
||||
_gate_idle()
|
||||
sm = request.app.state.serialmon
|
||||
await sm.suspend(serial)
|
||||
try:
|
||||
return await asyncio.to_thread(fn, *args)
|
||||
finally:
|
||||
await sm.resume(serial)
|
||||
|
||||
|
||||
# --- devices ---------------------------------------------------------------
|
||||
def _mount_devices(api: APIRouter) -> None:
|
||||
@api.get("/devices")
|
||||
async def list_devices(request: Request):
|
||||
return await rd.list_all(request.app.state.db)
|
||||
|
||||
@api.patch("/devices/{serial}")
|
||||
async def patch_device(serial: str, request: Request, body: dict = Body(...)):
|
||||
db, hub = request.app.state.db, request.app.state.hub
|
||||
await _device_or_404(db, serial)
|
||||
if "friendly_name" in body:
|
||||
dev = await rd.set_friendly_name(db, serial, body["friendly_name"])
|
||||
else:
|
||||
dev = await rd.get(db, serial)
|
||||
await hub.publish("device.update", dev)
|
||||
return dev
|
||||
|
||||
@api.put("/devices/{serial}/env")
|
||||
async def set_env(serial: str, request: Request, body: dict = Body(...)):
|
||||
db, hub = request.app.state.db, request.app.state.hub
|
||||
await _device_or_404(db, serial)
|
||||
env = body.get("env")
|
||||
# A provided env pins it; clearing it releases the pin to auto-detect.
|
||||
dev = await rd.set_env(db, serial, env, locked=env is not None)
|
||||
await hub.publish("device.update", dev)
|
||||
return dev
|
||||
|
||||
@api.post("/devices/{serial}/refresh")
|
||||
async def refresh(serial: str, request: Request):
|
||||
db, hub = request.app.state.db, request.app.state.hub
|
||||
row = await _device_or_404(db, serial)
|
||||
port = row.get("current_port")
|
||||
info = await _port_action(request, serial, mt_info.device_info, port)
|
||||
hw_model = info.get("hw_model")
|
||||
env = identity.env_for_hw_model(hw_model) if hw_model else None
|
||||
dev = await rd.update_enrichment(
|
||||
db,
|
||||
serial,
|
||||
node_num=info.get("my_node_num"),
|
||||
env=env,
|
||||
hw_model=hw_model,
|
||||
firmware_version=info.get("firmware_version"),
|
||||
region=info.get("region"),
|
||||
)
|
||||
await hub.publish("device.update", dev)
|
||||
return {"device": dev}
|
||||
|
||||
@api.get("/devices/{serial}/flash-stats")
|
||||
async def flash_stats(serial: str, request: Request):
|
||||
return await rf.comparison(request.app.state.db, serial)
|
||||
|
||||
@api.post("/devices/{serial}/flash")
|
||||
async def flash_device(serial: str, request: Request):
|
||||
db, hub = request.app.state.db, request.app.state.hub
|
||||
row = await _device_or_404(db, serial)
|
||||
env = control.env_for_device(row)
|
||||
port = row.get("current_port")
|
||||
if not env or not port:
|
||||
raise HTTPException(status_code=400, detail="no env/port resolved")
|
||||
loop = asyncio.get_running_loop()
|
||||
start = loop.time()
|
||||
result = await _port_action(
|
||||
request, serial, lambda: flash_lib.flash(env, port, confirm=True)
|
||||
)
|
||||
duration = round(loop.time() - start, 2)
|
||||
ok = result.get("exit_code") == 0
|
||||
fw = firmware.firmware_ref()
|
||||
await rf.record(
|
||||
db,
|
||||
device_serial=serial,
|
||||
env=env,
|
||||
fw_sha=fw.get("sha"),
|
||||
from_artifact=False,
|
||||
duration_s=duration,
|
||||
ok=ok,
|
||||
)
|
||||
if ok:
|
||||
await rd.record_flashed(db, serial, branch=fw.get("branch"), sha=fw.get("sha"))
|
||||
await hub.publish("device.update", await rd.get(db, serial))
|
||||
return {"ok": ok, "duration_s": duration, **result}
|
||||
|
||||
@api.post("/devices/{serial}/reboot")
|
||||
async def reboot(serial: str, request: Request):
|
||||
row = await _device_or_404(request.app.state.db, serial)
|
||||
return await _port_action(request, serial, admin.reboot, row.get("current_port"), True, 5)
|
||||
|
||||
@api.post("/devices/{serial}/factory-reset")
|
||||
async def factory_reset(serial: str, request: Request):
|
||||
row = await _device_or_404(request.app.state.db, serial)
|
||||
return await _port_action(
|
||||
request, serial, admin.factory_reset, row.get("current_port"), True
|
||||
)
|
||||
|
||||
@api.post("/devices/{serial}/send-text")
|
||||
async def send_text(serial: str, request: Request, body: dict = Body(...)):
|
||||
row = await _device_or_404(request.app.state.db, serial)
|
||||
text = body.get("text", "")
|
||||
return await _port_action(
|
||||
request, serial, admin.send_text, text, None, 0, False, row.get("current_port")
|
||||
)
|
||||
|
||||
@api.post("/devices/{serial}/inject-nodedb")
|
||||
async def inject_nodedb(serial: str, request: Request, body: dict = Body(...)):
|
||||
row = await _device_or_404(request.app.state.db, serial)
|
||||
size = int(body.get("size", 500))
|
||||
return await _port_action(request, serial, _inject, size, row.get("current_port"))
|
||||
|
||||
@api.get("/devices/{serial}/config")
|
||||
async def get_config(serial: str, request: Request, section: str | None = None):
|
||||
row = await _device_or_404(request.app.state.db, serial)
|
||||
return await _port_action(
|
||||
request, serial, admin.get_config, section, row.get("current_port")
|
||||
)
|
||||
|
||||
@api.put("/devices/{serial}/config")
|
||||
async def set_config(serial: str, request: Request, body: dict = Body(...)):
|
||||
row = await _device_or_404(request.app.state.db, serial)
|
||||
path = body.get("path")
|
||||
if not path:
|
||||
raise HTTPException(status_code=400, detail="missing config path")
|
||||
return await _port_action(
|
||||
request, serial, admin.set_config, path, body.get("value"), row.get("current_port")
|
||||
)
|
||||
|
||||
@api.get("/devices/{serial}/packets")
|
||||
async def device_packets(
|
||||
serial: str, request: Request, start: str = "-30m", max: int = 100
|
||||
):
|
||||
await _device_or_404(request.app.state.db, serial)
|
||||
# Recorder packets are mesh-wide, not keyed by USB port — return the
|
||||
# recent window so the per-device tab has live traffic to show.
|
||||
window = await asyncio.to_thread(
|
||||
lambda: log_query.packets_window(start, "now", max=max)
|
||||
)
|
||||
return {"packets": window.get("packets", [])}
|
||||
|
||||
@api.get("/devices/{serial}/test-results")
|
||||
async def device_test_results(serial: str, request: Request, limit: int = 100):
|
||||
rows = await rr.results_for_device(request.app.state.db, serial)
|
||||
return rows[:limit]
|
||||
|
||||
# --- per-device USB power (uhubctl) ----------------------------------
|
||||
@api.put("/devices/{serial}/hub-port")
|
||||
async def set_hub_port(serial: str, request: Request, body: dict = Body(...)):
|
||||
db, hub = request.app.state.db, request.app.state.hub
|
||||
await _device_or_404(db, serial)
|
||||
loc = body.get("location")
|
||||
port = body.get("port")
|
||||
dev = await rd.set_hub_port(
|
||||
db, serial, location=loc, port=int(port) if port is not None else None
|
||||
)
|
||||
await hub.publish("device.update", dev)
|
||||
return dev
|
||||
|
||||
@api.post("/devices/{serial}/locate")
|
||||
async def locate_device(serial: str, request: Request):
|
||||
db, hub = request.app.state.db, request.app.state.hub
|
||||
await _device_or_404(db, serial)
|
||||
try:
|
||||
res = await power.locate(db, serial)
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
if res["located"]:
|
||||
await hub.publish("device.update", res["device"])
|
||||
return res
|
||||
|
||||
@api.post("/devices/{serial}/power/{action}")
|
||||
async def power_action(serial: str, action: str, request: Request):
|
||||
db, hub = request.app.state.db, request.app.state.hub
|
||||
await _device_or_404(db, serial)
|
||||
if action not in ("on", "off", "cycle"):
|
||||
raise HTTPException(status_code=404, detail="unknown power action")
|
||||
_gate_idle()
|
||||
# Free the port from any live serial monitor before toggling VBUS.
|
||||
await request.app.state.serialmon.suspend(serial)
|
||||
try:
|
||||
result = await power.power_device(db, serial, action)
|
||||
except AmbiguousPort as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"error": str(exc), "candidates": exc.candidates},
|
||||
)
|
||||
except (NoPort, ValueError) as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
except RuntimeError as exc: # uhubctl errors (permissions, hub gone)
|
||||
raise HTTPException(status_code=502, detail=str(exc))
|
||||
finally:
|
||||
if action != "off":
|
||||
await request.app.state.serialmon.resume(serial)
|
||||
return result
|
||||
|
||||
|
||||
def _inject(size: int, port: str | None) -> dict:
|
||||
return fixtures.push_fake_nodedb(
|
||||
size, target="hardware", port=port, confirm=True, reboot_after=True
|
||||
)
|
||||
|
||||
|
||||
# --- cameras ---------------------------------------------------------------
|
||||
def _mount_cameras(api: APIRouter) -> None:
|
||||
@api.get("/cameras")
|
||||
async def list_cameras(request: Request):
|
||||
return await rc.list_all(request.app.state.db)
|
||||
|
||||
@api.get("/cameras/discover")
|
||||
async def discover_cameras(request: Request):
|
||||
# Indices already bound to a FleetSuite camera — don't re-open those.
|
||||
in_use = {
|
||||
str(c["device_index"])
|
||||
for c in await rc.list_all(request.app.state.db)
|
||||
if c.get("device_index") is not None
|
||||
}
|
||||
return await asyncio.to_thread(camera_stream.discover, in_use)
|
||||
|
||||
@api.post("/cameras")
|
||||
async def add_camera(request: Request, body: dict = Body(...)):
|
||||
db, hub = request.app.state.db, request.app.state.hub
|
||||
cid = await rc.add(
|
||||
db, name=body.get("name", "camera"), device_index=str(body.get("device_index", "0"))
|
||||
)
|
||||
cam = await rc.get(db, cid)
|
||||
await hub.publish("camera.update", cam)
|
||||
return cam
|
||||
|
||||
@api.delete("/cameras/{cid}", status_code=204)
|
||||
async def remove_camera(cid: int, request: Request):
|
||||
db, hub = request.app.state.db, request.app.state.hub
|
||||
await rc.remove(db, cid)
|
||||
await hub.publish("camera.update", {"id": cid, "deleted": True})
|
||||
|
||||
@api.post("/cameras/{cid}/assign")
|
||||
async def assign_camera(cid: int, request: Request, body: dict = Body(...)):
|
||||
db, hub = request.app.state.db, request.app.state.hub
|
||||
cam = await rc.assign(db, cid, body.get("device_serial"))
|
||||
if cam is None:
|
||||
raise HTTPException(status_code=404, detail="unknown camera")
|
||||
await hub.publish("camera.update", cam)
|
||||
return cam
|
||||
|
||||
@api.post("/cameras/{cid}/rotation")
|
||||
async def rotate_camera(cid: int, request: Request, body: dict = Body(...)):
|
||||
db, hub = request.app.state.db, request.app.state.hub
|
||||
cam = await rc.set_rotation(db, cid, int(body.get("rotation", 0)))
|
||||
if cam is None:
|
||||
raise HTTPException(status_code=404, detail="unknown camera")
|
||||
await hub.publish("camera.update", cam)
|
||||
return cam
|
||||
|
||||
@api.post("/cameras/{cid}/mirror")
|
||||
async def mirror_camera(cid: int, request: Request, body: dict = Body(...)):
|
||||
db, hub = request.app.state.db, request.app.state.hub
|
||||
cam = await rc.set_mirror(db, cid, bool(body.get("mirror", False)))
|
||||
if cam is None:
|
||||
raise HTTPException(status_code=404, detail="unknown camera")
|
||||
await hub.publish("camera.update", cam)
|
||||
return cam
|
||||
|
||||
@api.get("/cameras/{cid}/status")
|
||||
async def camera_status(cid: int, request: Request):
|
||||
cam = await rc.get(request.app.state.db, cid)
|
||||
if cam is None:
|
||||
raise HTTPException(status_code=404, detail="unknown camera")
|
||||
return await asyncio.to_thread(camera_stream.probe, str(cam.get("device_index")))
|
||||
|
||||
@api.get("/cameras/{cid}/stream.mjpg")
|
||||
async def camera_stream_ep(cid: int, request: Request):
|
||||
cam = await rc.get(request.app.state.db, cid)
|
||||
if cam is None:
|
||||
raise HTTPException(status_code=404, detail="unknown camera")
|
||||
probe = await asyncio.to_thread(camera_stream.probe, str(cam.get("device_index")))
|
||||
if not probe["ok"]:
|
||||
raise HTTPException(status_code=503, detail=probe["error"])
|
||||
return StreamingResponse(
|
||||
camera_stream.mjpeg(str(cam.get("device_index"))),
|
||||
media_type=f"multipart/x-mixed-replace; boundary={camera_stream.BOUNDARY}",
|
||||
)
|
||||
|
||||
|
||||
# --- firmware --------------------------------------------------------------
|
||||
def _mount_firmware(api: APIRouter) -> None:
|
||||
@api.get("/firmware")
|
||||
async def get_firmware():
|
||||
return await asyncio.to_thread(firmware.firmware_ref)
|
||||
|
||||
|
||||
# --- builds ----------------------------------------------------------------
|
||||
def _mount_builds(api: APIRouter) -> None:
|
||||
@api.get("/builds")
|
||||
async def list_builds(request: Request):
|
||||
db = request.app.state.db
|
||||
return {
|
||||
"docker": await asyncio.to_thread(builder.docker_available),
|
||||
"builds": await rb.list_all(db),
|
||||
}
|
||||
|
||||
@api.post("/builds")
|
||||
async def enqueue_builds(request: Request, body: dict = Body(default={})):
|
||||
db = request.app.state.db
|
||||
orch = request.app.state.orch
|
||||
fw = await asyncio.to_thread(firmware.firmware_ref)
|
||||
if not fw.get("available"):
|
||||
raise HTTPException(status_code=400, detail="no firmware checkout")
|
||||
sha, branch = fw["sha"], fw.get("branch")
|
||||
|
||||
envs = body.get("envs")
|
||||
if not envs:
|
||||
# Prebuild the envs every online, env-resolved device needs.
|
||||
envs = sorted(
|
||||
{control.env_for_device(d) for d in await rd.online_with_env(db)}
|
||||
- {None}
|
||||
)
|
||||
if not envs:
|
||||
return []
|
||||
return await orch.enqueue(
|
||||
list(envs), sha=sha, branch=branch, force=bool(body.get("force"))
|
||||
)
|
||||
|
||||
|
||||
# --- datadog ---------------------------------------------------------------
|
||||
def _mount_datadog(api: APIRouter) -> None:
|
||||
@api.get("/datadog")
|
||||
async def get_datadog(request: Request):
|
||||
return request.app.state.forwarder.status()
|
||||
|
||||
@api.put("/datadog")
|
||||
async def put_datadog(request: Request, body: dict = Body(...)):
|
||||
db = request.app.state.db
|
||||
fwd = request.app.state.forwarder
|
||||
cfg = await datadog.load_config(db)
|
||||
for key in ("enabled", "site", "scrub", "collector", "host", "ship_debug"):
|
||||
if key in body:
|
||||
setattr(cfg, key, body[key])
|
||||
# Only overwrite the key if a (non-empty) one was supplied.
|
||||
if body.get("api_key"):
|
||||
cfg.api_key = body["api_key"]
|
||||
await datadog.save_config(db, cfg)
|
||||
await fwd.reload()
|
||||
status = fwd.status()
|
||||
await request.app.state.hub.publish("datadog.update", status)
|
||||
return status
|
||||
|
||||
@api.post("/datadog/test")
|
||||
async def test_datadog(request: Request):
|
||||
fwd = request.app.state.forwarder
|
||||
await fwd.reload()
|
||||
return await asyncio.to_thread(fwd.test_key)
|
||||
|
||||
|
||||
# --- tests -----------------------------------------------------------------
|
||||
def _mount_tests(api: APIRouter) -> None:
|
||||
@api.get("/tests/status")
|
||||
async def tests_status():
|
||||
return test_runner.status()
|
||||
|
||||
@api.get("/tests/runs")
|
||||
async def tests_runs(request: Request):
|
||||
return await rr.list_runs(request.app.state.db)
|
||||
|
||||
@api.post("/tests/start")
|
||||
async def tests_start(request: Request, body: dict = Body(default={})):
|
||||
runner = request.app.state.runner
|
||||
try:
|
||||
return await runner.start(list(body.get("args", [])))
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc))
|
||||
|
||||
@api.post("/tests/stop", status_code=204)
|
||||
async def tests_stop(request: Request):
|
||||
await request.app.state.runner.stop()
|
||||
|
||||
|
||||
# --- native ----------------------------------------------------------------
|
||||
def _mount_native(api: APIRouter) -> None:
|
||||
@api.get("/native")
|
||||
async def native_info(request: Request):
|
||||
return await native.info(request.app.state.db)
|
||||
|
||||
@api.post("/native")
|
||||
async def native_create(request: Request, body: dict = Body(...)):
|
||||
db, hub = request.app.state.db, request.app.state.hub
|
||||
name = (body.get("name") or "").strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="missing name")
|
||||
try:
|
||||
dev = await native.create(db, name=name, tcp_port=int(body.get("tcp_port", 4403)))
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
await hub.publish("device.update", dev)
|
||||
return dev
|
||||
|
||||
@api.post("/native/{name}/{action}")
|
||||
async def native_lifecycle(name: str, action: str, request: Request):
|
||||
db, hub = request.app.state.db, request.app.state.hub
|
||||
if action not in ("start", "stop", "restart"):
|
||||
raise HTTPException(status_code=404, detail="unknown action")
|
||||
try:
|
||||
dev = await native.lifecycle(db, name, action)
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
await hub.publish("device.update", dev)
|
||||
return dev
|
||||
|
||||
@api.delete("/native/{name}", status_code=204)
|
||||
async def native_delete(name: str, request: Request):
|
||||
db, hub = request.app.state.db, request.app.state.hub
|
||||
await native.remove(db, name)
|
||||
await hub.publish("device.update", {"serial_number": f"native:{name}", "deleted": True})
|
||||
|
||||
|
||||
# --- boards ----------------------------------------------------------------
|
||||
def _mount_boards(api: APIRouter) -> None:
|
||||
@api.get("/boards")
|
||||
async def list_boards(query: str | None = None, architecture: str | None = None):
|
||||
return await asyncio.to_thread(
|
||||
boards.list_boards, architecture, False, query, None
|
||||
)
|
||||
|
||||
|
||||
# --- hubs (uhubctl) --------------------------------------------------------
|
||||
def _mount_hubs(api: APIRouter) -> None:
|
||||
@api.get("/hubs")
|
||||
async def list_hubs():
|
||||
if not power.available():
|
||||
return {"available": False, "hubs": []}
|
||||
try:
|
||||
hubs = await asyncio.to_thread(power.list_hubs)
|
||||
except RuntimeError as exc: # uhubctl present but failed (permissions)
|
||||
return {"available": True, "hubs": [], "error": str(exc)}
|
||||
return {"available": True, "hubs": hubs}
|
||||
|
||||
|
||||
# --- websocket -------------------------------------------------------------
|
||||
def _mount_ws(app: FastAPI) -> None:
|
||||
@app.websocket("/ws")
|
||||
async def ws(websocket: WebSocket):
|
||||
await websocket.accept()
|
||||
hub: Hub = websocket.app.state.hub
|
||||
sm = websocket.app.state.serialmon
|
||||
conn = Connection(send=websocket.send_json)
|
||||
hub.add(conn)
|
||||
# Track this peer's live serial monitors so we can release them on drop.
|
||||
serials: set[str] = set()
|
||||
try:
|
||||
while True:
|
||||
msg = await websocket.receive_json()
|
||||
action = msg.get("action")
|
||||
topic = msg.get("topic")
|
||||
if action == "subscribe" and topic:
|
||||
hub.subscribe(conn, topic)
|
||||
if topic.startswith("serial.") and topic not in serials:
|
||||
serials.add(topic)
|
||||
await sm.acquire(topic[len("serial.") :])
|
||||
elif action == "unsubscribe" and topic:
|
||||
hub.unsubscribe(conn, topic)
|
||||
if topic in serials:
|
||||
serials.discard(topic)
|
||||
await sm.release(topic[len("serial.") :])
|
||||
except Exception: # noqa: BLE001 - normal on client disconnect
|
||||
pass
|
||||
finally:
|
||||
hub.remove(conn)
|
||||
for topic in serials:
|
||||
await sm.release(topic[len("serial.") :])
|
||||
@@ -1,3 +0,0 @@
|
||||
"""SQLite registry for FleetSuite (aiosqlite). One ``Database`` owns the
|
||||
connection; the ``repo_*`` modules are stateless helpers that take it as their
|
||||
first argument."""
|
||||
@@ -1,185 +0,0 @@
|
||||
"""Thin async wrapper over a single aiosqlite connection.
|
||||
|
||||
``row_factory`` is set to ``aiosqlite.Row`` so every fetch behaves like a dict
|
||||
(``row["col"]``); the ``repo_*`` modules copy rows into plain dicts before
|
||||
handing them back so callers can add computed keys.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import aiosqlite
|
||||
|
||||
# --- schema -----------------------------------------------------------------
|
||||
# One file, created on connect. Plain `IF NOT EXISTS` — the registry is a cache
|
||||
# of discovered hardware and recorded history, not a source of truth, so a
|
||||
# wipe-and-rediscover is always safe.
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS devices (
|
||||
serial_number TEXT PRIMARY KEY,
|
||||
node_num INTEGER,
|
||||
friendly_name TEXT,
|
||||
hw_model TEXT,
|
||||
vid TEXT,
|
||||
pid TEXT,
|
||||
role TEXT,
|
||||
current_port TEXT,
|
||||
firmware_version TEXT,
|
||||
region TEXT,
|
||||
env TEXT,
|
||||
env_locked INTEGER NOT NULL DEFAULT 0,
|
||||
flashed_fw_branch TEXT,
|
||||
flashed_fw_sha TEXT,
|
||||
flashed_at REAL,
|
||||
online INTEGER NOT NULL DEFAULT 0,
|
||||
first_seen REAL NOT NULL DEFAULT 0,
|
||||
last_seen REAL NOT NULL DEFAULT 0,
|
||||
kind TEXT NOT NULL DEFAULT 'usb', -- 'usb' | 'native'
|
||||
tcp_port INTEGER,
|
||||
hub_location TEXT, -- uhubctl hub location (e.g. 1-1.3)
|
||||
hub_port INTEGER -- uhubctl port number on that hub
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cameras (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
type TEXT NOT NULL DEFAULT 'usb',
|
||||
device_index TEXT,
|
||||
backend TEXT,
|
||||
rotation INTEGER NOT NULL DEFAULT 0,
|
||||
mirror INTEGER NOT NULL DEFAULT 0,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
created_at REAL NOT NULL DEFAULT 0,
|
||||
device_serial TEXT,
|
||||
assigned_at REAL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS flash_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
device_serial TEXT NOT NULL,
|
||||
env TEXT,
|
||||
fw_sha TEXT,
|
||||
from_artifact INTEGER NOT NULL DEFAULT 0,
|
||||
duration_s REAL,
|
||||
ok INTEGER NOT NULL DEFAULT 1,
|
||||
ts REAL NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
started_at REAL NOT NULL DEFAULT 0,
|
||||
finished_at REAL,
|
||||
exit_code INTEGER,
|
||||
args TEXT,
|
||||
seed TEXT,
|
||||
fw_branch TEXT,
|
||||
fw_sha TEXT,
|
||||
fw_dirty INTEGER NOT NULL DEFAULT 0,
|
||||
passed INTEGER NOT NULL DEFAULT 0,
|
||||
failed INTEGER NOT NULL DEFAULT 0,
|
||||
skipped INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS results (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
run_id INTEGER NOT NULL,
|
||||
nodeid TEXT NOT NULL,
|
||||
tier TEXT,
|
||||
outcome TEXT,
|
||||
duration_s REAL,
|
||||
device_serial TEXT,
|
||||
longrepr TEXT,
|
||||
ts REAL NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_results_device ON results(device_serial);
|
||||
CREATE INDEX IF NOT EXISTS idx_results_run ON results(run_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS builds (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
env TEXT NOT NULL,
|
||||
fw_sha TEXT NOT NULL,
|
||||
fw_branch TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'queued',
|
||||
duration_s REAL,
|
||||
artifact_dir TEXT,
|
||||
error TEXT,
|
||||
created_at REAL NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_builds_key ON builds(env, fw_sha);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS settings (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def default_db_path() -> Path:
|
||||
"""Where the registry lives for a normal (non-test) run."""
|
||||
env = os.environ.get("MESHTASTIC_MCP_WEB_DB")
|
||||
if env:
|
||||
return Path(env)
|
||||
return Path.home() / ".meshtastic_mcp" / "fleetsuite.db"
|
||||
|
||||
|
||||
class Database:
|
||||
"""Owns one aiosqlite connection. ``await connect()`` before use, and
|
||||
``await close()`` when done. Helpers commit after every write — the access
|
||||
pattern is low-volume bench traffic, not a hot loop."""
|
||||
|
||||
def __init__(self, path: Path | str) -> None:
|
||||
self.path = Path(path)
|
||||
self._conn: aiosqlite.Connection | None = None
|
||||
|
||||
async def connect(self) -> "Database":
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._conn = await aiosqlite.connect(str(self.path))
|
||||
self._conn.row_factory = aiosqlite.Row
|
||||
await self._conn.execute("PRAGMA journal_mode=WAL;")
|
||||
await self._conn.executescript(SCHEMA)
|
||||
await self._migrate()
|
||||
await self._conn.commit()
|
||||
return self
|
||||
|
||||
async def _migrate(self) -> None:
|
||||
"""Additive column migrations for schema added after a db was first
|
||||
created. ``ALTER TABLE ADD COLUMN`` is idempotent here — a duplicate on
|
||||
a fresh db (already created with the column) is caught and ignored."""
|
||||
additions = (
|
||||
("cameras", "mirror", "INTEGER NOT NULL DEFAULT 0"),
|
||||
("devices", "hub_location", "TEXT"),
|
||||
("devices", "hub_port", "INTEGER"),
|
||||
)
|
||||
for table, col, decl in additions:
|
||||
try:
|
||||
await self._conn.execute(
|
||||
f"ALTER TABLE {table} ADD COLUMN {col} {decl}"
|
||||
)
|
||||
except Exception: # noqa: BLE001 - column already present
|
||||
pass
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._conn is not None:
|
||||
await self._conn.close()
|
||||
self._conn = None
|
||||
|
||||
@property
|
||||
def conn(self) -> aiosqlite.Connection:
|
||||
if self._conn is None:
|
||||
raise RuntimeError("Database.connect() not called")
|
||||
return self._conn
|
||||
|
||||
async def execute(self, sql: str, params: tuple = ()) -> aiosqlite.Cursor:
|
||||
cur = await self.conn.execute(sql, params)
|
||||
await self.conn.commit()
|
||||
return cur
|
||||
|
||||
async def fetchone(self, sql: str, params: tuple = ()):
|
||||
async with self.conn.execute(sql, params) as cur:
|
||||
return await cur.fetchone()
|
||||
|
||||
async def fetchall(self, sql: str, params: tuple = ()):
|
||||
async with self.conn.execute(sql, params) as cur:
|
||||
return await cur.fetchall()
|
||||
@@ -1,68 +0,0 @@
|
||||
"""Firmware-build ledger, keyed by (env, fw_sha). The orchestrator records one
|
||||
row per build attempt; ``get`` returns the latest for a key so a cache hit can
|
||||
be distinguished from a fresh queue."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
from .database import Database
|
||||
|
||||
_COLS = (
|
||||
"id, env, fw_sha, fw_branch, status, duration_s, artifact_dir, error, "
|
||||
"created_at"
|
||||
)
|
||||
|
||||
|
||||
def _to_dict(row) -> dict | None:
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
async def create(
|
||||
db: Database, *, env: str, fw_sha: str, fw_branch: str | None, status: str
|
||||
) -> int:
|
||||
cur = await db.execute(
|
||||
"INSERT INTO builds (env, fw_sha, fw_branch, status, created_at) "
|
||||
"VALUES (?,?,?,?,?)",
|
||||
(env, fw_sha, fw_branch, status, time.time()),
|
||||
)
|
||||
return cur.lastrowid
|
||||
|
||||
|
||||
async def set_status(
|
||||
db: Database,
|
||||
build_id: int,
|
||||
*,
|
||||
status: str,
|
||||
duration_s: float | None = None,
|
||||
artifact_dir: str | None = None,
|
||||
error: str | None = None,
|
||||
) -> dict | None:
|
||||
await db.execute(
|
||||
"UPDATE builds SET status=?, duration_s=?, artifact_dir=?, error=? "
|
||||
"WHERE id=?",
|
||||
(status, duration_s, artifact_dir, error, build_id),
|
||||
)
|
||||
return await get_by_id(db, build_id)
|
||||
|
||||
|
||||
async def get_by_id(db: Database, build_id: int) -> dict | None:
|
||||
row = await db.fetchone(f"SELECT {_COLS} FROM builds WHERE id=?", (build_id,))
|
||||
return _to_dict(row)
|
||||
|
||||
|
||||
async def get(db: Database, env: str, fw_sha: str) -> dict | None:
|
||||
"""Latest build row for a key, or None if it was never attempted."""
|
||||
row = await db.fetchone(
|
||||
f"SELECT {_COLS} FROM builds WHERE env=? AND fw_sha=? "
|
||||
"ORDER BY id DESC LIMIT 1",
|
||||
(env, fw_sha),
|
||||
)
|
||||
return _to_dict(row)
|
||||
|
||||
|
||||
async def list_all(db: Database, limit: int = 100) -> list[dict]:
|
||||
rows = await db.fetchall(
|
||||
f"SELECT {_COLS} FROM builds ORDER BY id DESC LIMIT ?", (limit,)
|
||||
)
|
||||
return [dict(r) for r in rows]
|
||||
@@ -1,81 +0,0 @@
|
||||
"""Camera registry. A camera is an independent entity (a USB capture device)
|
||||
that can be *assigned* to a device serial; rotation is a property of the camera
|
||||
mount, so it survives reassignment."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
from .database import Database
|
||||
|
||||
_COLS = (
|
||||
"id, name, type, device_index, backend, rotation, mirror, enabled, "
|
||||
"created_at, device_serial, assigned_at"
|
||||
)
|
||||
|
||||
|
||||
def _to_dict(row) -> dict | None:
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
def _normalize_rotation(rotation: int) -> int:
|
||||
"""Snap to the nearest quarter-turn and wrap into [0, 360)."""
|
||||
return int(round((rotation % 360) / 90.0)) * 90 % 360
|
||||
|
||||
|
||||
async def add(db: Database, *, name: str, device_index: str) -> int:
|
||||
cur = await db.execute(
|
||||
"INSERT INTO cameras (name, type, device_index, rotation, enabled, "
|
||||
"created_at) VALUES (?, 'usb', ?, 0, 1, ?)",
|
||||
(name, device_index, time.time()),
|
||||
)
|
||||
return cur.lastrowid
|
||||
|
||||
|
||||
async def get(db: Database, cid: int) -> dict | None:
|
||||
row = await db.fetchone(f"SELECT {_COLS} FROM cameras WHERE id=?", (cid,))
|
||||
return _to_dict(row)
|
||||
|
||||
|
||||
async def list_all(db: Database) -> list[dict]:
|
||||
rows = await db.fetchall(f"SELECT {_COLS} FROM cameras ORDER BY id")
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
async def remove(db: Database, cid: int) -> None:
|
||||
await db.execute("DELETE FROM cameras WHERE id=?", (cid,))
|
||||
|
||||
|
||||
async def assign(db: Database, cid: int, device_serial: str | None) -> dict | None:
|
||||
await db.execute(
|
||||
"UPDATE cameras SET device_serial=?, assigned_at=? WHERE id=?",
|
||||
(device_serial, time.time() if device_serial else None, cid),
|
||||
)
|
||||
return await get(db, cid)
|
||||
|
||||
|
||||
async def for_device(db: Database, serial: str) -> dict | None:
|
||||
row = await db.fetchone(
|
||||
f"SELECT {_COLS} FROM cameras WHERE device_serial=? LIMIT 1", (serial,)
|
||||
)
|
||||
return _to_dict(row)
|
||||
|
||||
|
||||
async def set_rotation(db: Database, cid: int, rotation: int) -> dict | None:
|
||||
await db.execute(
|
||||
"UPDATE cameras SET rotation=? WHERE id=?",
|
||||
(_normalize_rotation(rotation), cid),
|
||||
)
|
||||
return await get(db, cid)
|
||||
|
||||
|
||||
async def set_mirror(db: Database, cid: int, mirror: bool) -> dict | None:
|
||||
"""Horizontal flip — a property of the camera mount, like rotation."""
|
||||
await db.execute(
|
||||
"UPDATE cameras SET mirror=? WHERE id=?", (int(bool(mirror)), cid)
|
||||
)
|
||||
return await get(db, cid)
|
||||
|
||||
|
||||
async def set_backend(db: Database, cid: int, backend: str | None) -> None:
|
||||
await db.execute("UPDATE cameras SET backend=? WHERE id=?", (backend, cid))
|
||||
@@ -1,222 +0,0 @@
|
||||
"""Device registry — the heart of the harness's identity model.
|
||||
|
||||
A device is keyed by its USB serial number so its row (and everything attached
|
||||
to it — friendly name, camera, pinned env, flash/test history) *follows* it
|
||||
across ports and reboots. Discovery never deletes rows; it only flips ``online``
|
||||
so a unplugged device greys out instead of vanishing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
from ..services import identity
|
||||
from .database import Database
|
||||
|
||||
_COLS = (
|
||||
"serial_number, node_num, friendly_name, hw_model, vid, pid, role, "
|
||||
"current_port, firmware_version, region, env, env_locked, "
|
||||
"flashed_fw_branch, flashed_fw_sha, flashed_at, online, first_seen, "
|
||||
"last_seen, kind, tcp_port, hub_location, hub_port"
|
||||
)
|
||||
|
||||
|
||||
def _to_dict(row) -> dict | None:
|
||||
if row is None:
|
||||
return None
|
||||
d = dict(row)
|
||||
serial = d.get("serial_number") or ""
|
||||
d["has_stable_id"] = identity.has_stable_id(serial)
|
||||
# A device is "stale" once it has gone offline — the card stays but greys.
|
||||
d["stale"] = not bool(d.get("online"))
|
||||
return d
|
||||
|
||||
|
||||
async def get(db: Database, serial: str) -> dict | None:
|
||||
row = await db.fetchone(
|
||||
f"SELECT {_COLS} FROM devices WHERE serial_number=?", (serial,)
|
||||
)
|
||||
return _to_dict(row)
|
||||
|
||||
|
||||
async def list_all(db: Database) -> list[dict]:
|
||||
rows = await db.fetchall(f"SELECT {_COLS} FROM devices ORDER BY first_seen")
|
||||
return [_to_dict(r) for r in rows]
|
||||
|
||||
|
||||
async def upsert_from_discovery(
|
||||
db: Database,
|
||||
*,
|
||||
serial_number: str,
|
||||
current_port: str,
|
||||
vid: str | None,
|
||||
pid: str | None,
|
||||
role: str | None,
|
||||
) -> dict:
|
||||
"""Insert a freshly-seen device or refresh an existing one.
|
||||
|
||||
Returns the device dict with two transient flags the discovery loop uses to
|
||||
decide what to broadcast: ``_is_new`` and ``_port_changed``. Never clobbers
|
||||
operator-owned fields (friendly_name, pinned env) on re-discovery.
|
||||
"""
|
||||
now = time.time()
|
||||
existing = await get(db, serial_number)
|
||||
if existing is None:
|
||||
await db.execute(
|
||||
"INSERT INTO devices "
|
||||
"(serial_number, current_port, vid, pid, role, kind, online, "
|
||||
" first_seen, last_seen, env_locked) "
|
||||
"VALUES (?,?,?,?,?,'usb',1,?,?,0)",
|
||||
(serial_number, current_port, vid, pid, role, now, now),
|
||||
)
|
||||
row = await get(db, serial_number)
|
||||
row["_is_new"] = True
|
||||
row["_port_changed"] = False
|
||||
return row
|
||||
|
||||
port_changed = existing["current_port"] != current_port
|
||||
await db.execute(
|
||||
"UPDATE devices SET current_port=?, vid=?, pid=?, role=?, online=1, "
|
||||
"last_seen=? WHERE serial_number=?",
|
||||
(current_port, vid, pid, role, now, serial_number),
|
||||
)
|
||||
row = await get(db, serial_number)
|
||||
row["_is_new"] = False
|
||||
row["_port_changed"] = port_changed
|
||||
return row
|
||||
|
||||
|
||||
async def upsert_native(
|
||||
db: Database, *, name: str, tcp_port: int, online: bool = True
|
||||
) -> dict:
|
||||
"""A Docker ``meshtasticd`` node — surfaced as a device with a synthetic
|
||||
``native:<name>`` serial so the same UI/card machinery applies."""
|
||||
now = time.time()
|
||||
serial = f"native:{name}"
|
||||
port = f"tcp://127.0.0.1:{tcp_port}"
|
||||
existing = await get(db, serial)
|
||||
if existing is None:
|
||||
await db.execute(
|
||||
"INSERT INTO devices "
|
||||
"(serial_number, friendly_name, role, current_port, kind, "
|
||||
" tcp_port, online, first_seen, last_seen, env_locked) "
|
||||
"VALUES (?,?, 'native', ?, 'native', ?, ?, ?, ?, 0)",
|
||||
(serial, name, port, tcp_port, int(online), now, now),
|
||||
)
|
||||
else:
|
||||
await db.execute(
|
||||
"UPDATE devices SET tcp_port=?, current_port=?, online=?, "
|
||||
"last_seen=? WHERE serial_number=?",
|
||||
(tcp_port, port, int(online), now, serial),
|
||||
)
|
||||
return await get(db, serial)
|
||||
|
||||
|
||||
async def set_friendly_name(db: Database, serial: str, name: str) -> dict | None:
|
||||
await db.execute(
|
||||
"UPDATE devices SET friendly_name=? WHERE serial_number=?", (name, serial)
|
||||
)
|
||||
return await get(db, serial)
|
||||
|
||||
|
||||
async def set_env(
|
||||
db: Database, serial: str, env: str | None, *, locked: bool
|
||||
) -> dict | None:
|
||||
"""Pin (``locked=True``) or release (``locked=False``) the pio env. A pinned
|
||||
env is protected from auto-enrichment; releasing it lets detection win."""
|
||||
await db.execute(
|
||||
"UPDATE devices SET env=?, env_locked=? WHERE serial_number=?",
|
||||
(env, int(locked), serial),
|
||||
)
|
||||
return await get(db, serial)
|
||||
|
||||
|
||||
async def update_enrichment(
|
||||
db: Database,
|
||||
serial: str,
|
||||
*,
|
||||
node_num: int | None = None,
|
||||
env: str | None = None,
|
||||
hw_model: str | None = None,
|
||||
firmware_version: str | None = None,
|
||||
region: str | None = None,
|
||||
) -> dict | None:
|
||||
"""Apply data read off a connected device. ``env`` is only written when the
|
||||
operator has NOT pinned one (``env_locked=0``); the rest always apply."""
|
||||
row = await get(db, serial)
|
||||
if row is None:
|
||||
return None
|
||||
|
||||
sets = ["node_num=COALESCE(?, node_num)"]
|
||||
params: list = [node_num]
|
||||
for col, val in (
|
||||
("hw_model", hw_model),
|
||||
("firmware_version", firmware_version),
|
||||
("region", region),
|
||||
):
|
||||
sets.append(f"{col}=COALESCE(?, {col})")
|
||||
params.append(val)
|
||||
if env is not None and not row["env_locked"]:
|
||||
sets.append("env=?")
|
||||
params.append(env)
|
||||
|
||||
params.append(serial)
|
||||
await db.execute(
|
||||
f"UPDATE devices SET {', '.join(sets)} WHERE serial_number=?", tuple(params)
|
||||
)
|
||||
return await get(db, serial)
|
||||
|
||||
|
||||
async def set_hub_port(
|
||||
db: Database, serial: str, *, location: str | None, port: int | None
|
||||
) -> dict | None:
|
||||
"""Pin (or clear) which uhubctl hub location + port this device sits on, so
|
||||
power actions target the right physical port. Survives port/USB changes —
|
||||
it's the hub slot, not the tty."""
|
||||
await db.execute(
|
||||
"UPDATE devices SET hub_location=?, hub_port=? WHERE serial_number=?",
|
||||
(location, port, serial),
|
||||
)
|
||||
return await get(db, serial)
|
||||
|
||||
|
||||
async def record_flashed(
|
||||
db: Database, serial: str, *, branch: str | None, sha: str | None
|
||||
) -> None:
|
||||
await db.execute(
|
||||
"UPDATE devices SET flashed_fw_branch=?, flashed_fw_sha=?, flashed_at=? "
|
||||
"WHERE serial_number=?",
|
||||
(branch, sha, time.time(), serial),
|
||||
)
|
||||
|
||||
|
||||
async def mark_offline_except(db: Database, keep: set[str]) -> list[str]:
|
||||
"""Flip every currently-online device not in ``keep`` to offline. Returns
|
||||
the serials that *transitioned* (so the caller can broadcast just those)."""
|
||||
rows = await db.fetchall("SELECT serial_number FROM devices WHERE online=1")
|
||||
newly = [r["serial_number"] for r in rows if r["serial_number"] not in keep]
|
||||
for serial in newly:
|
||||
await db.execute(
|
||||
"UPDATE devices SET online=0 WHERE serial_number=?", (serial,)
|
||||
)
|
||||
return newly
|
||||
|
||||
|
||||
async def online_by_role(db: Database, role: str) -> dict | None:
|
||||
row = await db.fetchone(
|
||||
f"SELECT {_COLS} FROM devices WHERE online=1 AND role=? "
|
||||
"ORDER BY last_seen DESC LIMIT 1",
|
||||
(role,),
|
||||
)
|
||||
return _to_dict(row)
|
||||
|
||||
|
||||
async def online_with_env(db: Database) -> list[dict]:
|
||||
"""Online, real (USB) devices that have a resolved env — the candidates the
|
||||
test runner bakes per-board variant overrides from. Native nodes (no flash
|
||||
target) and un-enriched devices (no env yet) are excluded."""
|
||||
rows = await db.fetchall(
|
||||
f"SELECT {_COLS} FROM devices "
|
||||
"WHERE online=1 AND kind='usb' AND env IS NOT NULL"
|
||||
)
|
||||
return [_to_dict(r) for r in rows]
|
||||
@@ -1,58 +0,0 @@
|
||||
"""Flash-timing log. Each flash records whether it came from a prebuilt
|
||||
artifact or a from-scratch host rebuild, so the UI can show how much the
|
||||
artifact cache saves on a given device."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
from .database import Database
|
||||
|
||||
_COLS = "id, device_serial, env, fw_sha, from_artifact, duration_s, ok, ts"
|
||||
|
||||
|
||||
async def record(
|
||||
db: Database,
|
||||
*,
|
||||
device_serial: str,
|
||||
env: str | None,
|
||||
fw_sha: str | None,
|
||||
from_artifact: bool,
|
||||
duration_s: float,
|
||||
ok: bool,
|
||||
) -> None:
|
||||
await db.execute(
|
||||
"INSERT INTO flash_events "
|
||||
"(device_serial, env, fw_sha, from_artifact, duration_s, ok, ts) "
|
||||
"VALUES (?,?,?,?,?,?,?)",
|
||||
(
|
||||
device_serial,
|
||||
env,
|
||||
fw_sha,
|
||||
int(from_artifact),
|
||||
duration_s,
|
||||
int(ok),
|
||||
time.time(),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def _latest(db: Database, serial: str, from_artifact: bool) -> dict | None:
|
||||
row = await db.fetchone(
|
||||
f"SELECT {_COLS} FROM flash_events "
|
||||
"WHERE device_serial=? AND from_artifact=? AND ok=1 "
|
||||
"ORDER BY ts DESC LIMIT 1",
|
||||
(serial, int(from_artifact)),
|
||||
)
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
async def comparison(db: Database, serial: str) -> dict:
|
||||
"""Latest successful artifact-flash vs latest successful host-rebuild flash,
|
||||
plus the speedup ratio when both exist."""
|
||||
artifact = await _latest(db, serial, True)
|
||||
rebuild = await _latest(db, serial, False)
|
||||
speedup = None
|
||||
if artifact and rebuild and artifact["duration_s"]:
|
||||
speedup = round(rebuild["duration_s"] / artifact["duration_s"], 1)
|
||||
return {"artifact": artifact, "rebuild": rebuild, "speedup": speedup}
|
||||
@@ -1,98 +0,0 @@
|
||||
"""Test-run history. A run is a single pytest invocation; results are the
|
||||
per-test leaves, optionally attributed to the device they exercised."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
|
||||
from .database import Database
|
||||
|
||||
_RUN_COLS = (
|
||||
"id, started_at, finished_at, exit_code, args, seed, fw_branch, fw_sha, "
|
||||
"fw_dirty, passed, failed, skipped"
|
||||
)
|
||||
|
||||
|
||||
def _run_to_dict(row) -> dict | None:
|
||||
if row is None:
|
||||
return None
|
||||
d = dict(row)
|
||||
try:
|
||||
d["args"] = json.loads(d["args"]) if d.get("args") else []
|
||||
except (ValueError, TypeError):
|
||||
d["args"] = []
|
||||
return d
|
||||
|
||||
|
||||
async def create_run(
|
||||
db: Database,
|
||||
*,
|
||||
args: list[str],
|
||||
seed: str | None,
|
||||
fw_branch: str | None,
|
||||
fw_sha: str | None,
|
||||
fw_dirty: bool,
|
||||
) -> int:
|
||||
cur = await db.execute(
|
||||
"INSERT INTO runs (started_at, args, seed, fw_branch, fw_sha, fw_dirty) "
|
||||
"VALUES (?,?,?,?,?,?)",
|
||||
(time.time(), json.dumps(args or []), seed, fw_branch, fw_sha, int(fw_dirty)),
|
||||
)
|
||||
return cur.lastrowid
|
||||
|
||||
|
||||
async def get_run(db: Database, run_id: int) -> dict | None:
|
||||
row = await db.fetchone(f"SELECT {_RUN_COLS} FROM runs WHERE id=?", (run_id,))
|
||||
return _run_to_dict(row)
|
||||
|
||||
|
||||
async def list_runs(db: Database, limit: int = 50) -> list[dict]:
|
||||
rows = await db.fetchall(
|
||||
f"SELECT {_RUN_COLS} FROM runs ORDER BY id DESC LIMIT ?", (limit,)
|
||||
)
|
||||
return [_run_to_dict(r) for r in rows]
|
||||
|
||||
|
||||
async def finish_run(db: Database, run_id: int, *, exit_code: int | None) -> None:
|
||||
await db.execute(
|
||||
"UPDATE runs SET finished_at=?, exit_code=? WHERE id=?",
|
||||
(time.time(), exit_code, run_id),
|
||||
)
|
||||
|
||||
|
||||
async def add_result(
|
||||
db: Database,
|
||||
run_id: int,
|
||||
*,
|
||||
nodeid: str,
|
||||
tier: str | None,
|
||||
outcome: str,
|
||||
duration_s: float | None,
|
||||
device_serial: str | None,
|
||||
longrepr: str | None,
|
||||
) -> None:
|
||||
await db.execute(
|
||||
"INSERT INTO results "
|
||||
"(run_id, nodeid, tier, outcome, duration_s, device_serial, longrepr, ts) "
|
||||
"VALUES (?,?,?,?,?,?,?,?)",
|
||||
(run_id, nodeid, tier, outcome, duration_s, device_serial, longrepr, time.time()),
|
||||
)
|
||||
col = {"passed": "passed", "failed": "failed", "skipped": "skipped"}.get(outcome)
|
||||
if col:
|
||||
await db.execute(
|
||||
f"UPDATE runs SET {col}={col}+1 WHERE id=?", (run_id,)
|
||||
)
|
||||
|
||||
|
||||
async def results_for_device(db: Database, serial: str) -> list[dict]:
|
||||
"""Every recorded result for a device, newest first, joined to its run so
|
||||
each row carries the firmware sha it ran against."""
|
||||
rows = await db.fetchall(
|
||||
"SELECT r.id, r.run_id, r.nodeid, r.tier, r.outcome, r.duration_s, "
|
||||
"r.device_serial, r.longrepr, r.ts, runs.fw_sha, runs.fw_branch "
|
||||
"FROM results r JOIN runs ON r.run_id = runs.id "
|
||||
"WHERE r.device_serial=? ORDER BY r.id DESC",
|
||||
(serial,),
|
||||
)
|
||||
return [dict(r) for r in rows]
|
||||
@@ -1,26 +0,0 @@
|
||||
"""Tiny key/value store for JSON-serialisable config blobs (e.g. the Datadog
|
||||
forwarder settings)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from .database import Database
|
||||
|
||||
|
||||
async def set_json(db: Database, key: str, obj) -> None:
|
||||
await db.execute(
|
||||
"INSERT INTO settings (key, value) VALUES (?, ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
||||
(key, json.dumps(obj)),
|
||||
)
|
||||
|
||||
|
||||
async def get_json(db: Database, key: str) -> dict | None:
|
||||
row = await db.fetchone("SELECT value FROM settings WHERE key=?", (key,))
|
||||
if row is None or row["value"] is None:
|
||||
return None
|
||||
try:
|
||||
return json.loads(row["value"])
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
@@ -1,4 +0,0 @@
|
||||
"""FleetSuite service layer — identity reconciliation, control gating, the
|
||||
build orchestrator, the pytest runner, and the Datadog forwarder. These hold
|
||||
the harness's domain logic; the REST/WS layer in ``app.py`` is a thin shell
|
||||
over them."""
|
||||
@@ -1,180 +0,0 @@
|
||||
"""Firmware build orchestrator.
|
||||
|
||||
Builds are keyed by ``(env, fw_sha)`` and cached on disk under
|
||||
``$MESHTASTIC_MCP_ARTIFACT_DIR/<sha>/<env>``. ``enqueue`` returns immediately
|
||||
with a row per env (``status: queued`` for a fresh build, ``cached: True`` when
|
||||
an artifact already exists); the actual compile runs in a background task so the
|
||||
HTTP request never blocks. The compile itself is injectable (``build_fn``) so
|
||||
the queue/cache/dedup logic is testable without Docker or a real toolchain.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from ..db import repo_builds as rb
|
||||
|
||||
log = logging.getLogger("meshtastic_mcp.web.builder")
|
||||
|
||||
# Signature of an injectable build function: compile ``env`` into ``out``,
|
||||
# streaming progress through ``log_cb``; return True on success.
|
||||
BuildFn = Callable[[str, Path, Callable[[str], None]], bool]
|
||||
|
||||
|
||||
def _artifact_root() -> Path:
|
||||
return Path(
|
||||
os.environ.get(
|
||||
"MESHTASTIC_MCP_ARTIFACT_DIR",
|
||||
str(Path.home() / ".meshtastic_mcp" / "artifacts"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def artifact_dir(sha: str, env: str) -> Path:
|
||||
return _artifact_root() / sha / env
|
||||
|
||||
|
||||
def cached_artifact_dir(sha: str, env: str) -> Path | None:
|
||||
"""The cached artifact dir for a key if a successful build left one, else
|
||||
None. 'Successful' is proven by the presence of a flashable image."""
|
||||
d = artifact_dir(sha, env)
|
||||
if d.is_dir() and (
|
||||
any(d.glob("*.factory.bin")) or any(d.glob("*.bin")) or any(d.glob("*.uf2"))
|
||||
):
|
||||
return d
|
||||
return None
|
||||
|
||||
|
||||
def docker_available() -> bool:
|
||||
if not shutil.which("docker"):
|
||||
return False
|
||||
try:
|
||||
import subprocess
|
||||
|
||||
return (
|
||||
subprocess.run(
|
||||
["docker", "info"],
|
||||
capture_output=True,
|
||||
timeout=5,
|
||||
).returncode
|
||||
== 0
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
|
||||
|
||||
def default_build_fn(env: str, out: Path, log_cb: Callable[[str], None]) -> bool:
|
||||
"""Host pio build, copying the resulting images into ``out``. Best-effort —
|
||||
used when no build_fn is injected."""
|
||||
try:
|
||||
from meshtastic_mcp import config as mcfg, pio
|
||||
|
||||
root = mcfg.firmware_root()
|
||||
log_cb(f"pio run -e {env}")
|
||||
result = pio.run(["run", "-e", env], cwd=root, timeout=900, check=False)
|
||||
log_cb(pio.tail_lines(result.stdout + result.stderr, 40))
|
||||
if result.returncode != 0:
|
||||
return False
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
build_dir = root / ".pio" / "build" / env
|
||||
copied = 0
|
||||
for pattern in ("*.factory.bin", "*.bin", "*.uf2", "*.hex"):
|
||||
for f in build_dir.glob(pattern):
|
||||
shutil.copy2(f, out / f.name)
|
||||
copied += 1
|
||||
log_cb(f"copied {copied} artifact(s)")
|
||||
return copied > 0
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log_cb(f"build error: {exc}")
|
||||
return False
|
||||
|
||||
|
||||
class BuildOrchestrator:
|
||||
def __init__(self, db, hub, build_fn: BuildFn | None = None) -> None:
|
||||
self.db = db
|
||||
self.hub = hub
|
||||
self.build_fn = build_fn or default_build_fn
|
||||
self._inflight: dict[str, asyncio.Task] = {}
|
||||
|
||||
async def enqueue(
|
||||
self, envs: list[str], *, sha: str, branch: str | None, force: bool = False
|
||||
) -> list[dict]:
|
||||
"""Queue a build per env (skipping ones already cached or in flight).
|
||||
``force`` rebuilds even when an artifact is cached. Returns a status row
|
||||
per requested env."""
|
||||
results: list[dict] = []
|
||||
for env in envs:
|
||||
key = f"{env}@{sha}"
|
||||
|
||||
cached = None if force else cached_artifact_dir(sha, env)
|
||||
if cached is not None:
|
||||
row = await rb.get(self.db, env, sha)
|
||||
if row is None or row["status"] not in ("success", "cached"):
|
||||
bid = await rb.create(
|
||||
self.db, env=env, fw_sha=sha, fw_branch=branch, status="cached"
|
||||
)
|
||||
row = await rb.set_status(
|
||||
self.db,
|
||||
bid,
|
||||
status="cached",
|
||||
duration_s=0.0,
|
||||
artifact_dir=str(cached),
|
||||
)
|
||||
results.append({**row, "cached": True})
|
||||
await self.hub.publish("build.update", {**row, "cached": True})
|
||||
continue
|
||||
|
||||
if key in self._inflight and not self._inflight[key].done():
|
||||
row = await rb.get(self.db, env, sha)
|
||||
if row:
|
||||
results.append(row)
|
||||
continue
|
||||
|
||||
bid = await rb.create(
|
||||
self.db, env=env, fw_sha=sha, fw_branch=branch, status="queued"
|
||||
)
|
||||
row = await rb.get_by_id(self.db, bid)
|
||||
results.append(row)
|
||||
await self.hub.publish("build.update", row)
|
||||
self._inflight[key] = asyncio.create_task(
|
||||
self._run_build(bid, env, sha, key)
|
||||
)
|
||||
return results
|
||||
|
||||
async def _run_build(self, build_id: int, env: str, sha: str, key: str) -> None:
|
||||
loop = asyncio.get_running_loop()
|
||||
row = await rb.set_status(self.db, build_id, status="building")
|
||||
await self.hub.publish("build.update", row)
|
||||
|
||||
out = artifact_dir(sha, env)
|
||||
logs: list[str] = []
|
||||
|
||||
def log_cb(line: str) -> None:
|
||||
logs.append(line)
|
||||
self.hub.publish_threadsafe(
|
||||
"build.update", {"id": build_id, "env": env, "log": line}
|
||||
)
|
||||
|
||||
start = loop.time()
|
||||
try:
|
||||
ok = await asyncio.to_thread(self.build_fn, env, out, log_cb)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logs.append(f"exception: {exc}")
|
||||
ok = False
|
||||
duration = round(loop.time() - start, 2)
|
||||
|
||||
row = await rb.set_status(
|
||||
self.db,
|
||||
build_id,
|
||||
status="success" if ok else "failed",
|
||||
duration_s=duration,
|
||||
artifact_dir=str(out) if ok else None,
|
||||
error=None if ok else "\n".join(logs[-20:]) or "build failed",
|
||||
)
|
||||
await self.hub.publish("build.update", row)
|
||||
self._inflight.pop(key, None)
|
||||
@@ -1,170 +0,0 @@
|
||||
"""MJPEG camera streaming.
|
||||
|
||||
Opens a capture device by OpenCV index and yields a ``multipart/x-mixed-replace``
|
||||
JPEG stream for an ``<img>`` tag. Rotation is applied client-side (CSS), so the
|
||||
stream is raw. OpenCV (the ``[ui]`` extra) is optional — without it, ``probe``
|
||||
reports the error and the stream endpoint 503s instead of crashing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import AsyncIterator
|
||||
|
||||
log = logging.getLogger("meshtastic_mcp.web.camera_stream")
|
||||
|
||||
BOUNDARY = "frame"
|
||||
_FPS = 10.0
|
||||
_MAX_INDEX = 10 # don't probe past this
|
||||
|
||||
|
||||
def _import_cv2():
|
||||
try:
|
||||
import cv2 # type: ignore
|
||||
|
||||
try: # quiet the noisy "can't open camera" warnings during probing
|
||||
cv2.utils.logging.setLogLevel(cv2.utils.logging.LOG_LEVEL_SILENT)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return cv2
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
|
||||
|
||||
def _enumerate_names() -> list[tuple[int, str]]:
|
||||
"""Best-effort camera names from the OS, WITHOUT activating any device.
|
||||
Returns ``[(index, name), ...]``. The index is the OpenCV capture index
|
||||
(enumeration order on macOS, the videoN number on Linux)."""
|
||||
if sys.platform == "darwin":
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["system_profiler", "SPCameraDataType", "-json"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
items = json.loads(out.stdout).get("SPCameraDataType", [])
|
||||
return [(i, it.get("_name", f"camera {i}")) for i, it in enumerate(items)]
|
||||
except Exception: # noqa: BLE001
|
||||
return []
|
||||
if sys.platform.startswith("linux"):
|
||||
out: list[tuple[int, str]] = []
|
||||
for node in sorted(Path("/sys/class/video4linux").glob("video*")):
|
||||
try:
|
||||
idx = int(node.name[len("video") :])
|
||||
name = (node / "name").read_text().strip()
|
||||
out.append((idx, name or f"video{idx}"))
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
return out
|
||||
return []
|
||||
|
||||
|
||||
def _probe_resolution(cv2, index: int) -> dict | None:
|
||||
"""Open a capture index briefly to confirm it works + read its resolution.
|
||||
Activates the camera momentarily; returns None if it won't open/read."""
|
||||
try:
|
||||
cap = cv2.VideoCapture(index)
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
try:
|
||||
if not cap.isOpened():
|
||||
return None
|
||||
ok, _ = cap.read()
|
||||
if not ok:
|
||||
return None
|
||||
return {
|
||||
"width": int(cap.get(cv2.CAP_PROP_FRAME_WIDTH) or 0),
|
||||
"height": int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0),
|
||||
}
|
||||
finally:
|
||||
cap.release()
|
||||
|
||||
|
||||
def discover(skip: set[str] | None = None, probe_resolution: bool = True) -> dict:
|
||||
"""Enumerate attached cameras.
|
||||
|
||||
Name enumeration needs no OpenCV (works even without the ``[ui]`` extra), so
|
||||
discovery is useful before streaming is installed. When cv2 IS present, each
|
||||
not-in-use index is briefly opened to confirm it works and read its
|
||||
resolution. ``skip`` is the set of indices already bound to a FleetSuite
|
||||
camera — they're marked ``in_use`` and not re-opened (their stream owns them).
|
||||
"""
|
||||
skip = skip or set()
|
||||
cv2 = _import_cv2()
|
||||
named = _enumerate_names()
|
||||
|
||||
# If the OS gave us no names but cv2 is here, fall back to index probing.
|
||||
if not named and cv2 is not None:
|
||||
named = [(i, f"camera {i}") for i in range(_MAX_INDEX)]
|
||||
|
||||
cameras: list[dict] = []
|
||||
for index, name in named:
|
||||
if index >= _MAX_INDEX:
|
||||
continue
|
||||
entry: dict = {"index": index, "name": name, "in_use": str(index) in skip}
|
||||
if cv2 is not None and not entry["in_use"] and probe_resolution:
|
||||
res = _probe_resolution(cv2, index)
|
||||
if res is None:
|
||||
# Named by the OS but cv2 couldn't open it — surface, don't drop.
|
||||
entry["unavailable"] = True
|
||||
else:
|
||||
entry.update(res)
|
||||
cameras.append(entry)
|
||||
|
||||
return {
|
||||
"available": True,
|
||||
"cv2": cv2 is not None,
|
||||
"cameras": cameras,
|
||||
}
|
||||
|
||||
|
||||
def probe(device_index: str) -> dict:
|
||||
"""Can we open this device? Returns ``{ok, error}``."""
|
||||
cv2 = _import_cv2()
|
||||
if cv2 is None:
|
||||
return {"ok": False, "error": "opencv not installed (pip install -e '.[ui]')"}
|
||||
try:
|
||||
cap = cv2.VideoCapture(int(device_index))
|
||||
except (ValueError, TypeError):
|
||||
return {"ok": False, "error": f"invalid device index: {device_index!r}"}
|
||||
try:
|
||||
if not cap.isOpened():
|
||||
return {"ok": False, "error": "device did not open (in use or absent?)"}
|
||||
ok, _ = cap.read()
|
||||
return {"ok": bool(ok), "error": None if ok else "no frame from device"}
|
||||
finally:
|
||||
cap.release()
|
||||
|
||||
|
||||
async def mjpeg(device_index: str) -> AsyncIterator[bytes]:
|
||||
"""Async MJPEG frame generator. Reads happen in a worker thread so the event
|
||||
loop is never blocked on the camera."""
|
||||
cv2 = _import_cv2()
|
||||
if cv2 is None:
|
||||
return
|
||||
cap = await asyncio.to_thread(cv2.VideoCapture, int(device_index))
|
||||
try:
|
||||
if not cap.isOpened():
|
||||
return
|
||||
while True:
|
||||
ok, frame = await asyncio.to_thread(cap.read)
|
||||
if not ok:
|
||||
break
|
||||
enc_ok, buf = await asyncio.to_thread(cv2.imencode, ".jpg", frame)
|
||||
if enc_ok:
|
||||
jpg = buf.tobytes()
|
||||
yield (
|
||||
b"--" + BOUNDARY.encode() + b"\r\n"
|
||||
b"Content-Type: image/jpeg\r\n"
|
||||
b"Content-Length: " + str(len(jpg)).encode() + b"\r\n\r\n"
|
||||
+ jpg + b"\r\n"
|
||||
)
|
||||
await asyncio.sleep(1.0 / _FPS)
|
||||
finally:
|
||||
cap.release()
|
||||
@@ -1,33 +0,0 @@
|
||||
"""Device-control safety gate.
|
||||
|
||||
The central invariant: no ``connect()``-based action (flash, reboot, config,
|
||||
send-text, factory-reset, nodedb inject) may run while a test run holds the
|
||||
ports. Every control endpoint funnels through ``_ensure_idle`` /
|
||||
``_ensure_port_free`` first, which raise ``ControlBusy`` (surfaced as HTTP 409)
|
||||
when the runner is active.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from . import identity, test_runner
|
||||
|
||||
|
||||
class ControlBusy(RuntimeError):
|
||||
"""Raised when a control action is attempted while a test run is active."""
|
||||
|
||||
|
||||
def env_for_device(d: dict) -> str | None:
|
||||
"""The pio env to flash/bake for a device: its resolved env if it has one,
|
||||
otherwise the coarse role default."""
|
||||
return d.get("env") or identity.env_for_role(d.get("role"))
|
||||
|
||||
|
||||
def _ensure_idle() -> None:
|
||||
if test_runner.is_running():
|
||||
raise ControlBusy("a test run is in progress")
|
||||
|
||||
|
||||
def _ensure_port_free(port: str | None) -> None:
|
||||
# While a run is active it owns every port — nothing else may touch one.
|
||||
if test_runner.is_running():
|
||||
raise ControlBusy(f"port {port} is held by an active test run")
|
||||
@@ -1,290 +0,0 @@
|
||||
"""Datadog forwarder.
|
||||
|
||||
Pure mappers (``log_to_dd`` / ``telemetry_to_metrics``) turn recorder rows into
|
||||
dashboard-compatible Datadog payloads; ``_read_live`` is a cursor-based tail of
|
||||
the recorder's JSONL streams; ``DDConfig`` persists settings (with the API key
|
||||
masked on the way out). The shipping loop lives in ``DDForwarder``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from dataclasses import asdict, dataclass, fields
|
||||
from pathlib import Path
|
||||
|
||||
import logging
|
||||
|
||||
from ..db import repo_settings as rs
|
||||
from .scrub import Scrubber
|
||||
|
||||
log = logging.getLogger("meshtastic_mcp.web.datadog")
|
||||
|
||||
_ANSI = re.compile(r"\x1b\[[0-9;]*m")
|
||||
|
||||
_DDSOURCE = "meshtastic-firmware"
|
||||
|
||||
_STATUS = {
|
||||
"ERROR": "error",
|
||||
"CRIT": "error",
|
||||
"CRITICAL": "error",
|
||||
"WARN": "warning",
|
||||
"WARNING": "warning",
|
||||
"INFO": "info",
|
||||
"DEBUG": "debug",
|
||||
"TRACE": "debug",
|
||||
"HEAP": "debug",
|
||||
}
|
||||
|
||||
|
||||
def _strip_ansi(s: str) -> str:
|
||||
return _ANSI.sub("", s or "")
|
||||
|
||||
|
||||
def _status_for(level: str | None) -> str:
|
||||
if not level:
|
||||
# Un-leveled output is a panic/backtrace — treat as error.
|
||||
return "error"
|
||||
return _STATUS.get(level.upper(), "info")
|
||||
|
||||
|
||||
# --- log mapping ------------------------------------------------------------
|
||||
def log_to_dd(
|
||||
rec: dict,
|
||||
*,
|
||||
host: str,
|
||||
base_tags: list[str],
|
||||
port_tags: dict[str, list[str]],
|
||||
scrubber: Scrubber,
|
||||
ship_debug: bool,
|
||||
) -> dict | None:
|
||||
"""Map a recorder log row to a Datadog log intake payload.
|
||||
|
||||
Returns None for a DEBUG line when ``ship_debug`` is False. Un-leveled lines
|
||||
(panics/backtraces) always ship.
|
||||
"""
|
||||
level = rec.get("level")
|
||||
if level and level.upper() == "DEBUG" and not ship_debug:
|
||||
return None
|
||||
|
||||
message = scrubber.scrub(_strip_ansi(rec.get("line", "")))
|
||||
|
||||
tags = list(base_tags)
|
||||
port = rec.get("port")
|
||||
if port:
|
||||
tags.append(f"port:{port}")
|
||||
tags.extend(port_tags.get(port, []))
|
||||
if level:
|
||||
tags.append(f"level:{level.lower()}")
|
||||
tag = rec.get("tag")
|
||||
if tag:
|
||||
tags.append(f"thread:{tag.lower()}")
|
||||
|
||||
payload = {
|
||||
"ddsource": _DDSOURCE,
|
||||
"service": _DDSOURCE,
|
||||
"hostname": host,
|
||||
"message": message,
|
||||
"ddtags": ",".join(tags),
|
||||
"status": _status_for(level),
|
||||
}
|
||||
if level is not None:
|
||||
payload["level"] = level
|
||||
if rec.get("ts") is not None:
|
||||
payload["timestamp"] = int(round(rec["ts"] * 1000))
|
||||
if rec.get("heap_free") is not None:
|
||||
payload["heap_free"] = rec["heap_free"]
|
||||
return payload
|
||||
|
||||
|
||||
# --- metric mapping ---------------------------------------------------------
|
||||
def telemetry_to_metrics(
|
||||
rec: dict,
|
||||
*,
|
||||
host: str,
|
||||
base_tags: list[str],
|
||||
port_tags: dict[str, list[str]],
|
||||
) -> list[dict]:
|
||||
"""Map a telemetry row to Datadog GAUGE series (one per numeric field).
|
||||
Boolean and string fields are dropped."""
|
||||
variant = rec.get("variant", "")
|
||||
fields_ = rec.get("fields", {}) or {}
|
||||
ts = int(rec.get("ts", 0) or 0)
|
||||
|
||||
tags = list(base_tags) + [f"variant:{variant}"]
|
||||
port = rec.get("port")
|
||||
if port:
|
||||
tags.extend(port_tags.get(port, []))
|
||||
|
||||
out: list[dict] = []
|
||||
for key, value in fields_.items():
|
||||
# bool is a subclass of int — exclude it explicitly.
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
continue
|
||||
out.append(
|
||||
{
|
||||
"metric": f"mesh.{variant}.{key}",
|
||||
"type": 3, # GAUGE
|
||||
"points": [{"timestamp": ts, "value": float(value)}],
|
||||
"resources": [{"type": "host", "name": host}],
|
||||
"tags": list(tags),
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
# --- cursor-based JSONL tail ------------------------------------------------
|
||||
def _read_live(path: Path, cursor: dict, max_lines: int) -> tuple[list[dict], dict]:
|
||||
"""Read newly-appended complete JSON lines from ``path``.
|
||||
|
||||
``cursor`` is ``{"ino": int, "pos": int}``. A partial trailing line (no
|
||||
newline yet) is left for the next cycle. A cursor whose inode no longer
|
||||
matches, or whose position is past EOF (rotation/truncation), resets to the
|
||||
start. Returns ``(rows, next_cursor)``.
|
||||
"""
|
||||
path = Path(path)
|
||||
try:
|
||||
st = path.stat()
|
||||
except OSError:
|
||||
return [], dict(cursor)
|
||||
ino = st.st_ino
|
||||
|
||||
pos = cursor.get("pos", 0) if cursor.get("ino") == ino else 0
|
||||
if pos > st.st_size: # truncated or rotated under us
|
||||
pos = 0
|
||||
|
||||
with open(path, "rb") as fh:
|
||||
fh.seek(pos)
|
||||
data = fh.read()
|
||||
|
||||
last_nl = data.rfind(b"\n")
|
||||
if last_nl == -1:
|
||||
# No complete line available — leave the cursor where it was.
|
||||
return [], {"ino": ino, "pos": pos}
|
||||
|
||||
complete = data[: last_nl + 1]
|
||||
consumed = pos + len(complete)
|
||||
|
||||
rows: list[dict] = []
|
||||
for raw in complete.split(b"\n"):
|
||||
if not raw.strip():
|
||||
continue
|
||||
try:
|
||||
rows.append(json.loads(raw))
|
||||
except ValueError:
|
||||
continue
|
||||
if len(rows) >= max_lines:
|
||||
break
|
||||
return rows, {"ino": ino, "pos": consumed}
|
||||
|
||||
|
||||
# --- intake host derivation -------------------------------------------------
|
||||
def _browser_intake_origin(site: str) -> str:
|
||||
"""The RUM/browser-logs intake origin for a Datadog site.
|
||||
|
||||
``us5.datadoghq.com`` → ``https://browser-intake-us5-datadoghq.com``
|
||||
``datadoghq.eu`` → ``https://browser-intake-datadoghq.eu``
|
||||
"""
|
||||
head, _, tld = site.rpartition(".")
|
||||
head = head.replace(".", "-")
|
||||
return f"https://browser-intake-{head}.{tld}" if head else f"https://browser-intake-{tld}"
|
||||
|
||||
|
||||
def _logs_intake_url(site: str) -> str:
|
||||
return f"https://http-intake.logs.{site}/api/v2/logs"
|
||||
|
||||
|
||||
def _metrics_intake_url(site: str) -> str:
|
||||
return f"https://api.{site}/api/v2/series"
|
||||
|
||||
|
||||
# --- config -----------------------------------------------------------------
|
||||
@dataclass
|
||||
class DDConfig:
|
||||
enabled: bool = False
|
||||
api_key: str = ""
|
||||
site: str = "datadoghq.com"
|
||||
scrub: str = "coarse"
|
||||
collector: str = ""
|
||||
host: str = ""
|
||||
ship_debug: bool = False
|
||||
|
||||
def masked(self) -> dict:
|
||||
"""Config for the UI — the API key is never exposed, only a hint and a
|
||||
client-token flag (Datadog client tokens start with ``pub``)."""
|
||||
return {
|
||||
"enabled": self.enabled,
|
||||
"site": self.site,
|
||||
"scrub": self.scrub,
|
||||
"collector": self.collector,
|
||||
"host": self.host,
|
||||
"ship_debug": self.ship_debug,
|
||||
"has_key": bool(self.api_key),
|
||||
"key_hint": self.api_key[-4:] if self.api_key else "",
|
||||
"is_client_token": self.api_key.startswith("pub") if self.api_key else False,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: dict | None) -> "DDConfig":
|
||||
d = d or {}
|
||||
allowed = {f.name for f in fields(cls)}
|
||||
return cls(**{k: v for k, v in d.items() if k in allowed})
|
||||
|
||||
|
||||
async def load_config(db) -> DDConfig:
|
||||
return DDConfig.from_dict(await rs.get_json(db, "datadog"))
|
||||
|
||||
|
||||
async def save_config(db, cfg: DDConfig) -> None:
|
||||
await rs.set_json(db, "datadog", asdict(cfg))
|
||||
|
||||
|
||||
# --- forwarder (runtime) ----------------------------------------------------
|
||||
def _recorder_dir() -> Path:
|
||||
return Path(os.environ.get("MESHTASTIC_MCP_RECORDER_DIR", ".mtlog"))
|
||||
|
||||
|
||||
class DDForwarder:
|
||||
"""Background loop that tails the recorder streams and ships to Datadog.
|
||||
Started/reconfigured from the ``/api/datadog`` routes; a no-op until a key
|
||||
and ``enabled`` are set."""
|
||||
|
||||
def __init__(self, db, hub) -> None:
|
||||
self.db = db
|
||||
self.hub = hub
|
||||
self.cfg = DDConfig()
|
||||
self.stats = {
|
||||
"running": False,
|
||||
"sent_logs": 0,
|
||||
"sent_metrics": 0,
|
||||
"cycles": 0,
|
||||
"last_error": None,
|
||||
"last_cycle_ts": None,
|
||||
}
|
||||
self._cursors: dict[str, dict] = {}
|
||||
|
||||
def status(self) -> dict:
|
||||
return {"config": self.cfg.masked(), "stats": dict(self.stats)}
|
||||
|
||||
async def reload(self) -> None:
|
||||
self.cfg = await load_config(self.db)
|
||||
|
||||
def test_key(self) -> dict:
|
||||
"""Validate the configured API key against Datadog. Best-effort —
|
||||
returns ``{ok, error}`` and never raises."""
|
||||
if not self.cfg.api_key:
|
||||
return {"ok": False, "error": "no API key configured"}
|
||||
try:
|
||||
import requests
|
||||
|
||||
resp = requests.get(
|
||||
f"https://api.{self.cfg.site}/api/v1/validate",
|
||||
headers={"DD-API-KEY": self.cfg.api_key},
|
||||
timeout=10,
|
||||
)
|
||||
if resp.ok and resp.json().get("valid"):
|
||||
return {"ok": True, "error": None}
|
||||
return {"ok": False, "error": f"validation failed ({resp.status_code})"}
|
||||
except Exception as exc: # noqa: BLE001 - surfaced to the UI
|
||||
return {"ok": False, "error": str(exc)}
|
||||
@@ -1,199 +0,0 @@
|
||||
"""Background USB discovery loop.
|
||||
|
||||
Polls the serial bus, reconciles each likely-Meshtastic device into the
|
||||
registry (keyed by stable serial, surrogate key otherwise), flips vanished
|
||||
devices offline, and broadcasts the deltas on ``device.update``.
|
||||
|
||||
Auto-enrichment: when a device is newly seen (or hops ports, or hasn't been
|
||||
read yet), the loop fires a one-shot ``device_info`` in the background to sniff
|
||||
its firmware version, hw_model → exact pio env, region, and node num — so those
|
||||
populate on their own at plug-in, FleetLog-style. It is gated hard for safety:
|
||||
skipped entirely while a test run holds the ports, serialized so only one
|
||||
device is connected at a time, the device's live serial monitor is suspended
|
||||
for the moment of the connect, pinned envs are never clobbered, and failures
|
||||
back off instead of re-hammering every poll. Disable with
|
||||
``MESHTASTIC_MCP_AUTO_ENRICH=0``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
from meshtastic_mcp import devices as devices_lib
|
||||
|
||||
from ..db import repo_devices as rd
|
||||
from . import identity
|
||||
|
||||
log = logging.getLogger("meshtastic_mcp.web.discovery")
|
||||
|
||||
POLL_SECONDS = 4.0
|
||||
ENRICH_BACKOFF_S = 60.0 # after a failed connect, wait this long before retrying
|
||||
|
||||
|
||||
class DeviceDiscovery:
|
||||
def __init__(self, db, hub, serialmon=None) -> None:
|
||||
self.db = db
|
||||
self.hub = hub
|
||||
self.serialmon = serialmon
|
||||
self.auto_enrich = os.environ.get("MESHTASTIC_MCP_AUTO_ENRICH", "1") != "0"
|
||||
self._task: asyncio.Task | None = None
|
||||
self._enrich_lock = asyncio.Lock() # one device on the wire at a time
|
||||
self._enriched: dict[str, str] = {} # serial -> port last enriched at
|
||||
self._failed: dict[str, float] = {} # serial -> monotonic time to retry after
|
||||
self._enriching: set[str] = set() # in-flight, to dedupe schedules
|
||||
|
||||
def start(self) -> None:
|
||||
if self._task is None:
|
||||
self._task = asyncio.create_task(self._loop())
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._task is not None:
|
||||
self._task.cancel()
|
||||
self._task = None
|
||||
|
||||
async def scan_once(self) -> None:
|
||||
"""One discovery pass. Runs the (blocking) enumeration in a thread."""
|
||||
try:
|
||||
found = await asyncio.to_thread(devices_lib.list_devices)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.debug("discovery enumeration failed: %s", exc)
|
||||
return
|
||||
|
||||
seen: set[str] = set()
|
||||
for dev in found:
|
||||
if dev.get("blacklisted") or not dev.get("likely_meshtastic", False):
|
||||
continue
|
||||
key, _stable = identity.device_key(dev)
|
||||
role = identity.role_for_vid(dev.get("vid"))
|
||||
seen.add(key)
|
||||
row = await rd.upsert_from_discovery(
|
||||
self.db,
|
||||
serial_number=key,
|
||||
current_port=dev.get("port"),
|
||||
vid=dev.get("vid"),
|
||||
pid=dev.get("pid"),
|
||||
role=role,
|
||||
)
|
||||
changed = bool(row.pop("_is_new", False)) | bool(
|
||||
row.pop("_port_changed", False)
|
||||
)
|
||||
if changed:
|
||||
await self.hub.publish("device.update", row)
|
||||
self._maybe_enrich(row, changed)
|
||||
|
||||
# Keep native nodes alive across scans — they aren't on the USB bus.
|
||||
natives = {
|
||||
d["serial_number"]
|
||||
for d in await rd.list_all(self.db)
|
||||
if d.get("kind") == "native"
|
||||
}
|
||||
newly_offline = await rd.mark_offline_except(self.db, seen | natives)
|
||||
for serial in newly_offline:
|
||||
self._enriched.pop(serial, None) # re-verify if it comes back
|
||||
row = await rd.get(self.db, serial)
|
||||
if row:
|
||||
await self.hub.publish("device.update", row)
|
||||
|
||||
# --- auto-enrichment --------------------------------------------------
|
||||
def _maybe_enrich(self, row: dict, changed: bool) -> None:
|
||||
"""Decide whether a discovered device needs a background enrichment and,
|
||||
if so, schedule one. Cheap, synchronous gate; the actual connect happens
|
||||
in :meth:`_enrich`."""
|
||||
if not self.auto_enrich:
|
||||
return
|
||||
serial = row.get("serial_number")
|
||||
if not serial or row.get("kind") == "native":
|
||||
return
|
||||
# Lazy import avoids a module-load cycle (test_runner ← control ← ...).
|
||||
from . import test_runner
|
||||
|
||||
if test_runner.is_running():
|
||||
return
|
||||
if serial in self._enriching:
|
||||
return
|
||||
retry_at = self._failed.get(serial)
|
||||
if retry_at is not None and time.monotonic() < retry_at:
|
||||
return
|
||||
# Enrich once per (serial, port). A completed-but-incomplete read sets a
|
||||
# backoff (handled in _enrich), so we never reconnect every poll.
|
||||
needs = changed or self._enriched.get(serial) != row.get("current_port")
|
||||
if needs:
|
||||
asyncio.create_task(self._enrich(serial))
|
||||
|
||||
async def _enrich(self, serial: str) -> None:
|
||||
from meshtastic_mcp import info as mt_info
|
||||
|
||||
from . import test_runner
|
||||
|
||||
if serial in self._enriching:
|
||||
return
|
||||
self._enriching.add(serial)
|
||||
try:
|
||||
async with self._enrich_lock: # serialize: one port open at a time
|
||||
if test_runner.is_running():
|
||||
return
|
||||
row = await rd.get(self.db, serial)
|
||||
if (
|
||||
row is None
|
||||
or not row.get("online")
|
||||
or row.get("kind") == "native"
|
||||
):
|
||||
return
|
||||
port = row.get("current_port")
|
||||
if not port:
|
||||
return
|
||||
|
||||
# Free the port from any live serial monitor for the connect.
|
||||
if self.serialmon is not None:
|
||||
await self.serialmon.suspend(serial)
|
||||
try:
|
||||
info = await asyncio.to_thread(mt_info.device_info, port)
|
||||
finally:
|
||||
if self.serialmon is not None:
|
||||
await self.serialmon.resume(serial)
|
||||
|
||||
hw_model = info.get("hw_model")
|
||||
env = identity.env_for_hw_model(hw_model) if hw_model else None
|
||||
updated = await rd.update_enrichment(
|
||||
self.db,
|
||||
serial,
|
||||
node_num=info.get("my_node_num"),
|
||||
env=env,
|
||||
hw_model=str(hw_model) if hw_model else None,
|
||||
firmware_version=info.get("firmware_version"),
|
||||
region=info.get("region"),
|
||||
)
|
||||
if info.get("firmware_version"):
|
||||
# Full read — terminal for this port.
|
||||
self._enriched[serial] = port
|
||||
self._failed.pop(serial, None)
|
||||
log.info(
|
||||
"enriched %s: fw=%s hw=%s env=%s",
|
||||
serial,
|
||||
info.get("firmware_version"),
|
||||
hw_model,
|
||||
env,
|
||||
)
|
||||
else:
|
||||
# Connected but metadata wasn't ready yet — retry after backoff.
|
||||
self._failed[serial] = time.monotonic() + ENRICH_BACKOFF_S
|
||||
if updated:
|
||||
await self.hub.publish("device.update", updated)
|
||||
except Exception as exc: # noqa: BLE001 - a flaky device shouldn't kill the loop
|
||||
self._failed[serial] = time.monotonic() + ENRICH_BACKOFF_S
|
||||
log.debug("enrichment of %s failed (backing off): %s", serial, exc)
|
||||
finally:
|
||||
self._enriching.discard(serial)
|
||||
|
||||
async def _loop(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
await self.scan_once()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.debug("discovery loop error: %s", exc)
|
||||
await asyncio.sleep(POLL_SECONDS)
|
||||
@@ -1,54 +0,0 @@
|
||||
"""Current firmware git ref — branch, sha, dirty flag, and the tip commit's
|
||||
subject/date. Read straight from git in the firmware root; degrades to
|
||||
``{"available": False}`` outside a checkout."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _root() -> Path:
|
||||
try:
|
||||
from meshtastic_mcp import config as mcfg
|
||||
|
||||
return mcfg.firmware_root()
|
||||
except Exception: # noqa: BLE001
|
||||
return Path.cwd()
|
||||
|
||||
|
||||
def _git(*args: str) -> str | None:
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["git", *args],
|
||||
cwd=str(_root()),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
if out.returncode != 0:
|
||||
return None
|
||||
return out.stdout.strip()
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
|
||||
|
||||
def firmware_ref() -> dict:
|
||||
sha = _git("rev-parse", "HEAD")
|
||||
if not sha:
|
||||
return {"available": False}
|
||||
branch = _git("rev-parse", "--abbrev-ref", "HEAD")
|
||||
status = _git("status", "--porcelain")
|
||||
subject = _git("log", "-1", "--pretty=%s")
|
||||
committed_at = _git("log", "-1", "--pretty=%cI")
|
||||
return {
|
||||
"available": True,
|
||||
"branch": branch,
|
||||
"sha": sha,
|
||||
"short_sha": sha[:7],
|
||||
"dirty": bool(status),
|
||||
"subject": subject,
|
||||
"committed_at": committed_at,
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
"""Device identity reconciliation.
|
||||
|
||||
Two jobs: (1) map a USB VID to a coarse *role* and a role to a default pio
|
||||
*env*, and resolve the precise env from a board's hw_model when we know it;
|
||||
(2) derive a *stable key* for a device so a unit with a real serial number is
|
||||
tracked across ports, while a serial-less unit gets a (port-derived) surrogate
|
||||
that is explicitly NOT stable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
|
||||
from meshtastic_mcp import boards
|
||||
|
||||
# USB vendor IDs we recognise → coarse role. Case-insensitive; compared as the
|
||||
# lowercased hex string (e.g. "0x239a").
|
||||
_VID_ROLE = {
|
||||
"0x239a": "nrf52", # Adafruit / RAK nRF52840
|
||||
"0x303a": "esp32s3", # Espressif native USB
|
||||
"0x10c4": "esp32s3", # Silicon Labs CP210x UART bridge
|
||||
"0x1a86": "esp32s3", # WCH CH340/CH9102 UART bridge
|
||||
}
|
||||
|
||||
# Coarse role → the default pio env to fall back on when we can't resolve the
|
||||
# exact board variant from its hw_model.
|
||||
_ROLE_ENV = {
|
||||
"nrf52": "rak4631",
|
||||
"esp32s3": "heltec-v3",
|
||||
}
|
||||
|
||||
_NOSERIAL_PREFIX = "noserial:"
|
||||
|
||||
|
||||
def role_for_vid(vid: str | None) -> str | None:
|
||||
if not vid:
|
||||
return None
|
||||
return _VID_ROLE.get(vid.lower())
|
||||
|
||||
|
||||
def env_for_role(role: str | None) -> str | None:
|
||||
if not role:
|
||||
return None
|
||||
return _ROLE_ENV.get(role)
|
||||
|
||||
|
||||
def env_for_hw_model(hw_model: str | None) -> str | None:
|
||||
"""Resolve the exact pio env for a hardware model slug (e.g. ``HELTEC_V4``).
|
||||
|
||||
Prefers the *base* env (``heltec-v4``) over decorated variants
|
||||
(``heltec-v4-tft``): when several envs declare the same hw_model slug, the
|
||||
one whose name is the canonical slugification wins, else the shortest.
|
||||
Returns None for an unknown slug.
|
||||
"""
|
||||
if not hw_model:
|
||||
return None
|
||||
target = hw_model.upper()
|
||||
candidates = [
|
||||
b["env"]
|
||||
for b in boards.list_boards()
|
||||
if (b.get("hw_model_slug") or "").upper() == target and b.get("env")
|
||||
]
|
||||
if not candidates:
|
||||
return None
|
||||
canonical = hw_model.lower().replace("_", "-")
|
||||
if canonical in candidates:
|
||||
return canonical
|
||||
return min(candidates, key=len)
|
||||
|
||||
|
||||
def device_key(d: dict) -> tuple[str, bool]:
|
||||
"""Return ``(key, is_stable)`` for a discovered-device dict.
|
||||
|
||||
A real serial number is a stable key. Without one, we synthesise a
|
||||
``noserial:<hash>`` surrogate from vid/pid/port so the device is still
|
||||
addressable within a session — but it is NOT stable across replug.
|
||||
"""
|
||||
serial = d.get("serial_number")
|
||||
if serial:
|
||||
return str(serial), True
|
||||
raw = f"{d.get('vid')}:{d.get('pid')}:{d.get('port')}"
|
||||
digest = hashlib.sha1(raw.encode()).hexdigest()[:12]
|
||||
return f"{_NOSERIAL_PREFIX}{digest}", False
|
||||
|
||||
|
||||
def has_stable_id(key: str | None) -> bool:
|
||||
return bool(key) and not str(key).startswith(_NOSERIAL_PREFIX)
|
||||
@@ -1,86 +0,0 @@
|
||||
"""Native (Docker ``meshtasticd``) node lifecycle.
|
||||
|
||||
Each native node is a container publishing the simulator's TCP API (4403) on a
|
||||
host port, mirrored into the device registry as a ``native:<name>`` device so
|
||||
the same card/UI applies. Best-effort: every Docker call is guarded and the
|
||||
service reports ``docker: False`` rather than raising when Docker is absent.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import shutil
|
||||
|
||||
from ..db import repo_devices as rd
|
||||
from . import builder
|
||||
|
||||
log = logging.getLogger("meshtastic_mcp.web.native")
|
||||
|
||||
IMAGE = "meshtastic/meshtasticd:latest"
|
||||
_PREFIX = "fleetsuite-native-"
|
||||
|
||||
|
||||
def docker_available() -> bool:
|
||||
return builder.docker_available()
|
||||
|
||||
|
||||
async def _docker(*args: str, timeout: int = 30) -> tuple[int, str]:
|
||||
if not shutil.which("docker"):
|
||||
return 127, "docker not found"
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"docker",
|
||||
*args,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
)
|
||||
try:
|
||||
out, _ = await asyncio.wait_for(proc.communicate(), timeout=timeout)
|
||||
except asyncio.TimeoutError:
|
||||
proc.kill()
|
||||
return 124, "docker timed out"
|
||||
return proc.returncode, out.decode(errors="replace")
|
||||
|
||||
|
||||
def _container(name: str) -> str:
|
||||
return f"{_PREFIX}{name}"
|
||||
|
||||
|
||||
async def info(db) -> dict:
|
||||
nodes = [
|
||||
d for d in await rd.list_all(db) if d.get("kind") == "native"
|
||||
]
|
||||
return {"docker": docker_available(), "image": IMAGE, "nodes": nodes}
|
||||
|
||||
|
||||
async def create(db, *, name: str, tcp_port: int) -> dict:
|
||||
code, out = await _docker(
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
_container(name),
|
||||
"-p",
|
||||
f"{tcp_port}:4403",
|
||||
IMAGE,
|
||||
)
|
||||
if code != 0:
|
||||
raise RuntimeError(f"docker run failed: {out.strip()[:200]}")
|
||||
return await rd.upsert_native(db, name=name, tcp_port=tcp_port, online=True)
|
||||
|
||||
|
||||
async def lifecycle(db, name: str, action: str) -> dict:
|
||||
verb = {"start": "start", "stop": "stop", "restart": "restart"}.get(action)
|
||||
if verb is None:
|
||||
raise ValueError(f"unknown action: {action}")
|
||||
code, out = await _docker(verb, _container(name))
|
||||
if code != 0:
|
||||
raise RuntimeError(f"docker {verb} failed: {out.strip()[:200]}")
|
||||
online = action != "stop"
|
||||
row = await rd.get(db, f"native:{name}")
|
||||
tcp_port = row.get("tcp_port") if row else 4403
|
||||
return await rd.upsert_native(db, name=name, tcp_port=tcp_port, online=online)
|
||||
|
||||
|
||||
async def remove(db, name: str) -> None:
|
||||
await _docker("rm", "-f", _container(name))
|
||||
await db.execute("DELETE FROM devices WHERE serial_number=?", (f"native:{name}",))
|
||||
@@ -1,120 +0,0 @@
|
||||
"""Per-node USB power control via uhubctl.
|
||||
|
||||
Each bench node sits on a PPPS-capable (per-port-power-switching) hub. uhubctl
|
||||
can toggle VBUS on a given ``(hub_location, port)``, but its port listing only
|
||||
exposes VID:PID — not the USB serial — so two same-VID nRF52s can't be told
|
||||
apart from the hub side. We therefore track the mapping explicitly on the device
|
||||
row (``hub_location`` / ``hub_port``):
|
||||
|
||||
* ``locate`` auto-binds a device when exactly one PPPS port matches its VID;
|
||||
* an ambiguous match (two identical boards) is surfaced for the operator to
|
||||
pick the right slot, which is then pinned and survives replug/reboot.
|
||||
|
||||
Power actions resolve through that mapping (falling back to a unique VID match)
|
||||
and are gated by the run-safety lock like any other control action.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from meshtastic_mcp import uhubctl
|
||||
|
||||
from ..db import repo_devices as rd
|
||||
|
||||
log = logging.getLogger("meshtastic_mcp.web.power")
|
||||
|
||||
_ACTIONS = {"on": uhubctl.power_on, "off": uhubctl.power_off, "cycle": uhubctl.cycle}
|
||||
|
||||
|
||||
class AmbiguousPort(RuntimeError):
|
||||
"""More than one PPPS port matches the device's VID — operator must pick."""
|
||||
|
||||
def __init__(self, candidates: list[dict]) -> None:
|
||||
super().__init__("ambiguous hub port — assign one manually")
|
||||
self.candidates = candidates
|
||||
|
||||
|
||||
class NoPort(RuntimeError):
|
||||
"""No controllable (PPPS) hub port hosts this device."""
|
||||
|
||||
|
||||
def available() -> bool:
|
||||
try:
|
||||
from meshtastic_mcp import config as mcfg
|
||||
|
||||
binary = mcfg.uhubctl_bin()
|
||||
if binary and Path(str(binary)).exists():
|
||||
return True
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return shutil.which("uhubctl") is not None
|
||||
|
||||
|
||||
def _vid_int(device: dict) -> int | None:
|
||||
vid = device.get("vid")
|
||||
if not vid:
|
||||
return None
|
||||
try:
|
||||
return int(str(vid), 16)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def list_hubs() -> list[dict]:
|
||||
"""Parsed uhubctl hubs (with PPPS flag + per-port attachments)."""
|
||||
return uhubctl.list_hubs()
|
||||
|
||||
|
||||
def candidates_for(device: dict) -> list[dict]:
|
||||
"""All PPPS ``(location, port)`` slots whose attached VID matches the
|
||||
device, as ``[{"location", "port"}]``."""
|
||||
vid = _vid_int(device)
|
||||
if vid is None:
|
||||
return []
|
||||
return [
|
||||
{"location": loc, "port": port}
|
||||
for loc, port in uhubctl.find_port_for_vid(vid)
|
||||
]
|
||||
|
||||
|
||||
async def locate(db, serial: str) -> dict:
|
||||
"""Try to auto-bind the device to its hub port. Returns the resolved
|
||||
mapping, or the candidate list when ambiguous so the UI can prompt."""
|
||||
device = await rd.get(db, serial)
|
||||
if device is None:
|
||||
raise LookupError(serial)
|
||||
cands = await asyncio.to_thread(candidates_for, device)
|
||||
if len(cands) == 1:
|
||||
dev = await rd.set_hub_port(
|
||||
db, serial, location=cands[0]["location"], port=cands[0]["port"]
|
||||
)
|
||||
return {"located": True, "device": dev, "candidates": cands}
|
||||
return {"located": False, "device": device, "candidates": cands}
|
||||
|
||||
|
||||
async def power_device(db, serial: str, action: str) -> dict:
|
||||
"""Run a power action against the device's hub port. Resolves the pinned
|
||||
mapping first, falling back to a unique VID match; raises ``AmbiguousPort``
|
||||
/ ``NoPort`` when the slot can't be determined."""
|
||||
if action not in _ACTIONS:
|
||||
raise ValueError(f"unknown power action: {action}")
|
||||
device = await rd.get(db, serial)
|
||||
if device is None:
|
||||
raise LookupError(serial)
|
||||
|
||||
loc = device.get("hub_location")
|
||||
port = device.get("hub_port")
|
||||
if loc is None or port is None:
|
||||
cands = await asyncio.to_thread(candidates_for, device)
|
||||
if not cands:
|
||||
raise NoPort("no PPPS hub port hosts this device — assign one manually")
|
||||
if len(cands) > 1:
|
||||
raise AmbiguousPort(cands)
|
||||
loc, port = cands[0]["location"], cands[0]["port"]
|
||||
|
||||
result = await asyncio.to_thread(_ACTIONS[action], loc, port)
|
||||
return {"location": loc, "port": port, **result}
|
||||
@@ -1,44 +0,0 @@
|
||||
"""Coordinate scrubber for outbound telemetry/logs.
|
||||
|
||||
Three modes:
|
||||
off — pass through unchanged
|
||||
coarse — round decimal lat/lon to 1 dp, truncate 1e-7 integer coords to the
|
||||
nearest million, drop NMEA sentence bodies
|
||||
redact — replace any coordinate value with ``<scrubbed>``
|
||||
|
||||
Only coordinate-named keys are touched, so a decimal like ``latency=12.5`` is
|
||||
left alone.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
# Decimal coordinate: lat/lon/latitude/longitude = <signed decimal>. Longest
|
||||
# names first so "latitude" isn't half-matched as "lat".
|
||||
_DEC = re.compile(r"\b(latitude|longitude|lat|lon)=(-?\d+\.\d+)")
|
||||
# Protobuf 1e-7 integer coordinate: same names with an _i suffix = <signed int>.
|
||||
_INT = re.compile(r"\b(latitude_i|longitude_i|lat_i|lon_i)=(-?\d+)")
|
||||
# NMEA sentence ("$GPGGA,...") — drop everything after the sentence type.
|
||||
_NMEA = re.compile(r"(\$G[A-Z]{4}),\S*")
|
||||
|
||||
|
||||
class Scrubber:
|
||||
def __init__(self, mode: str = "coarse") -> None:
|
||||
self.mode = (mode or "off").lower()
|
||||
|
||||
def scrub(self, text: str) -> str:
|
||||
if self.mode == "off" or not text:
|
||||
return text
|
||||
if self.mode == "redact":
|
||||
text = _DEC.sub(lambda m: f"{m.group(1)}=<scrubbed>", text)
|
||||
text = _INT.sub(lambda m: f"{m.group(1)}=<scrubbed>", text)
|
||||
text = _NMEA.sub(r"\1,<scrubbed>", text)
|
||||
return text
|
||||
# coarse
|
||||
text = _DEC.sub(lambda m: f"{m.group(1)}={round(float(m.group(2)), 1)}", text)
|
||||
text = _INT.sub(
|
||||
lambda m: f"{m.group(1)}={int(float(m.group(2)) / 1e6) * 1_000_000}", text
|
||||
)
|
||||
text = _NMEA.sub(r"\1,<scrubbed>", text)
|
||||
return text
|
||||
@@ -1,142 +0,0 @@
|
||||
"""Live serial monitors, one per device, multiplexed onto the ``serial.<serial>``
|
||||
WebSocket topic.
|
||||
|
||||
A monitor is reference-counted by subscriber: the first client to open a
|
||||
device's Serial tab spawns a pyserial reader thread that republishes each line;
|
||||
the last to leave tears it down. Direct pyserial is used (not ``pio device
|
||||
monitor``, whose miniterm backend requires a controlling TTY and crashes when
|
||||
run headless under the server). Because the reader holds the USB port, any
|
||||
control action ``suspend``s it for the duration and ``resume``s after, so the
|
||||
port is never double-opened.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import threading
|
||||
|
||||
import serial as pyserial
|
||||
|
||||
from meshtastic_mcp import connection
|
||||
|
||||
from ..db import repo_devices as rd
|
||||
|
||||
log = logging.getLogger("meshtastic_mcp.web.serial_monitor")
|
||||
|
||||
BAUD = 115200
|
||||
|
||||
|
||||
class _Monitor:
|
||||
def __init__(self) -> None:
|
||||
self.refs = 0
|
||||
self.suspended = False
|
||||
self.stop: threading.Event | None = None
|
||||
self.thread: threading.Thread | None = None
|
||||
|
||||
|
||||
class SerialMonitor:
|
||||
def __init__(self, db, hub) -> None:
|
||||
self.db = db
|
||||
self.hub = hub
|
||||
self._mons: dict[str, _Monitor] = {}
|
||||
|
||||
def _topic(self, serial: str) -> str:
|
||||
return f"serial.{serial}"
|
||||
|
||||
async def acquire(self, serial: str) -> None:
|
||||
mon = self._mons.setdefault(serial, _Monitor())
|
||||
mon.refs += 1
|
||||
if mon.refs == 1 and not mon.suspended:
|
||||
await self._open(serial, mon)
|
||||
|
||||
async def release(self, serial: str) -> None:
|
||||
mon = self._mons.get(serial)
|
||||
if mon is None:
|
||||
return
|
||||
mon.refs = max(0, mon.refs - 1)
|
||||
if mon.refs == 0:
|
||||
await self._close(mon)
|
||||
self._mons.pop(serial, None)
|
||||
|
||||
async def suspend(self, serial: str) -> None:
|
||||
"""Free the port for a control action (no-op if not monitored)."""
|
||||
mon = self._mons.get(serial)
|
||||
if mon is None:
|
||||
return
|
||||
mon.suspended = True
|
||||
await self._close(mon)
|
||||
|
||||
async def resume(self, serial: str) -> None:
|
||||
mon = self._mons.get(serial)
|
||||
if mon is None:
|
||||
return
|
||||
mon.suspended = False
|
||||
if mon.refs > 0 and mon.thread is None:
|
||||
await self._open(serial, mon)
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
for mon in list(self._mons.values()):
|
||||
await self._close(mon)
|
||||
self._mons.clear()
|
||||
|
||||
async def _open(self, serial: str, mon: _Monitor) -> None:
|
||||
row = await rd.get(self.db, serial)
|
||||
if row is None or row.get("kind") == "native":
|
||||
return # native nodes are TCP — nothing to monitor on the USB bus
|
||||
port = row.get("current_port")
|
||||
if not port or connection.is_tcp_port(port):
|
||||
return
|
||||
mon.stop = threading.Event()
|
||||
mon.thread = threading.Thread(
|
||||
target=self._read_loop, args=(serial, port, mon.stop), daemon=True
|
||||
)
|
||||
mon.thread.start()
|
||||
|
||||
async def _close(self, mon: _Monitor) -> None:
|
||||
if mon.stop is not None:
|
||||
mon.stop.set()
|
||||
thread = mon.thread
|
||||
mon.thread = None
|
||||
mon.stop = None
|
||||
if thread is not None:
|
||||
await asyncio.to_thread(thread.join, 2.0)
|
||||
|
||||
def _read_loop(self, serial: str, port: str, stop: threading.Event) -> None:
|
||||
"""Runs in a worker thread; publishes lines via the hub's thread-safe
|
||||
path. Reads with a short timeout so ``stop`` is honoured promptly."""
|
||||
topic = self._topic(serial)
|
||||
try:
|
||||
ser = pyserial.Serial(port, BAUD, timeout=0.5)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self.hub.publish_threadsafe(topic, {"line": f"— cannot open {port}: {exc} —"})
|
||||
return
|
||||
self.hub.publish_threadsafe(topic, {"line": f"— monitor opened on {port} —"})
|
||||
buf = b""
|
||||
try:
|
||||
while not stop.is_set():
|
||||
try:
|
||||
data = ser.read(256)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self.hub.publish_threadsafe(topic, {"line": f"— read error: {exc} —"})
|
||||
break
|
||||
if not data:
|
||||
continue
|
||||
buf += data
|
||||
while b"\n" in buf:
|
||||
raw, buf = buf.split(b"\n", 1)
|
||||
text = raw.decode("utf-8", "replace").rstrip("\r")
|
||||
if not text:
|
||||
continue
|
||||
# The meshtastic CDC carries protobuf API frames interleaved
|
||||
# with text debug logs. Drop lines that are mostly undecodable
|
||||
# bytes (a protobuf frame) — decoded text logs render with ANSI.
|
||||
bad = text.count("�")
|
||||
if bad and bad > len(text) * 0.2:
|
||||
continue
|
||||
self.hub.publish_threadsafe(topic, {"line": text})
|
||||
finally:
|
||||
try:
|
||||
ser.close()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
@@ -1,267 +0,0 @@
|
||||
"""The pytest runner.
|
||||
|
||||
``resolve_env_overrides`` and ``is_running`` are the pure pieces the safety
|
||||
gate and the run-launcher depend on. ``TestRunner`` drives an actual pytest
|
||||
subprocess: it bakes per-board env overrides, tails ``pytest-reportlog`` JSONL
|
||||
for live per-test progress, and streams stdout/stderr + firmware logs over the
|
||||
hub.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
log = logging.getLogger("meshtastic_mcp.web.test_runner")
|
||||
|
||||
# Tiers the UI knows about, in display order. A nodeid maps to a tier by its
|
||||
# path under tests/ (directory name, or "bake"/"unit" for top-level files).
|
||||
TIERS = (
|
||||
"bake",
|
||||
"unit",
|
||||
"mesh",
|
||||
"telemetry",
|
||||
"monitor",
|
||||
"fleet",
|
||||
"admin",
|
||||
"provisioning",
|
||||
)
|
||||
|
||||
# Module-level run state. A single harness runs one suite at a time.
|
||||
_state: dict = {"running": False, "run_id": None, "exit_code": None, "proc": None}
|
||||
|
||||
|
||||
def is_running() -> bool:
|
||||
return bool(_state.get("running"))
|
||||
|
||||
|
||||
def status() -> dict:
|
||||
return {
|
||||
"running": _state["running"],
|
||||
"run_id": _state["run_id"],
|
||||
"exit_code": _state["exit_code"],
|
||||
}
|
||||
|
||||
|
||||
def resolve_env_overrides(rows: list[dict]) -> dict[str, str]:
|
||||
"""From the online, env-resolved devices, bake one
|
||||
``MESHTASTIC_MCP_ENV_<ROLE>=<env>`` override per role. Rows without a role
|
||||
or env are skipped (so native/TCP nodes never become a flash target)."""
|
||||
overrides: dict[str, str] = {}
|
||||
for row in rows:
|
||||
role = row.get("role")
|
||||
env = row.get("env")
|
||||
if not role or not env:
|
||||
continue
|
||||
overrides[f"MESHTASTIC_MCP_ENV_{role.upper()}"] = env
|
||||
return overrides
|
||||
|
||||
|
||||
def tier_for(nodeid: str) -> str:
|
||||
"""Derive a tier from a pytest nodeid path."""
|
||||
path = nodeid.split("::", 1)[0]
|
||||
parts = path.split("/")
|
||||
if "tests" in parts:
|
||||
rest = parts[parts.index("tests") + 1 :]
|
||||
if rest:
|
||||
seg = rest[0]
|
||||
if seg.endswith(".py"):
|
||||
return "bake" if "bake" in seg else "unit"
|
||||
return seg
|
||||
return "unit"
|
||||
|
||||
|
||||
def _split_nodeid(nodeid: str) -> tuple[str, str]:
|
||||
path, _, name = nodeid.partition("::")
|
||||
return path, name or nodeid
|
||||
|
||||
|
||||
class TestRunner:
|
||||
"""Owns the live pytest subprocess + its reportlog tail. One per app."""
|
||||
|
||||
def __init__(self, db, hub) -> None:
|
||||
self.db = db
|
||||
self.hub = hub
|
||||
self._task: asyncio.Task | None = None
|
||||
|
||||
async def start(self, args: list[str]) -> dict:
|
||||
from . import firmware # local import to avoid a cycle at module load
|
||||
|
||||
if is_running():
|
||||
raise RuntimeError("a test run is already in progress")
|
||||
|
||||
fw = firmware.firmware_ref()
|
||||
from ..db import repo_devices as rd
|
||||
|
||||
overrides = resolve_env_overrides(await rd.online_with_env(self.db))
|
||||
|
||||
from ..db import repo_runs as rr
|
||||
|
||||
run_id = await rr.create_run(
|
||||
self.db,
|
||||
args=args,
|
||||
seed=str(int(time.time())),
|
||||
fw_branch=fw.get("branch"),
|
||||
fw_sha=fw.get("sha"),
|
||||
fw_dirty=bool(fw.get("dirty")),
|
||||
)
|
||||
_state.update(running=True, run_id=run_id, exit_code=None)
|
||||
await self.hub.publish("test.progress", {"type": "run_started", "run_id": run_id})
|
||||
self._task = asyncio.create_task(self._drive(run_id, args, overrides))
|
||||
return status()
|
||||
|
||||
async def stop(self) -> None:
|
||||
proc = _state.get("proc")
|
||||
if proc and proc.returncode is None:
|
||||
try:
|
||||
proc.terminate()
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
async def _drive(self, run_id: int, args: list[str], overrides: dict) -> None:
|
||||
from .. import config as _cfg # noqa: F401
|
||||
from ..db import repo_runs as rr
|
||||
from meshtastic_mcp import config as mcfg
|
||||
|
||||
exit_code = None
|
||||
report = Path(tempfile.gettempdir()) / f"fleetsuite-report-{run_id}.jsonl"
|
||||
report.unlink(missing_ok=True)
|
||||
try:
|
||||
root = mcfg.firmware_root() / "mcp-server"
|
||||
except Exception: # noqa: BLE001
|
||||
root = Path.cwd()
|
||||
|
||||
env = dict(os.environ)
|
||||
env.update(overrides)
|
||||
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"pytest",
|
||||
"-p",
|
||||
"no:cacheprovider",
|
||||
f"--report-log={report}",
|
||||
"-v",
|
||||
*args,
|
||||
]
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
cwd=str(root),
|
||||
env=env,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
_state["proc"] = proc
|
||||
tail = asyncio.create_task(self._tail_report(run_id, report))
|
||||
await asyncio.gather(
|
||||
self._pump(proc.stdout, "stdout"),
|
||||
self._pump(proc.stderr, "stderr"),
|
||||
)
|
||||
exit_code = await proc.wait()
|
||||
await asyncio.sleep(0.2) # let the reportlog flush
|
||||
tail.cancel()
|
||||
except FileNotFoundError as exc:
|
||||
await self.hub.publish(
|
||||
"test.stdout", {"line": f"failed to launch pytest: {exc}", "source": "stderr"}
|
||||
)
|
||||
exit_code = 127
|
||||
finally:
|
||||
_state.update(running=False, exit_code=exit_code, proc=None)
|
||||
await rr.finish_run(self.db, run_id, exit_code=exit_code)
|
||||
await self.hub.publish(
|
||||
"test.progress", {"type": "run_finished", "exit_code": exit_code}
|
||||
)
|
||||
report.unlink(missing_ok=True)
|
||||
|
||||
async def _pump(self, stream, source: str) -> None:
|
||||
if stream is None:
|
||||
return
|
||||
while True:
|
||||
raw = await stream.readline()
|
||||
if not raw:
|
||||
break
|
||||
line = raw.decode(errors="replace").rstrip("\n")
|
||||
await self.hub.publish("test.stdout", {"line": line, "source": source})
|
||||
|
||||
async def _tail_report(self, run_id: int, report: Path) -> None:
|
||||
"""Follow the reportlog JSONL and translate entries into progress frames
|
||||
+ persisted results."""
|
||||
from ..db import repo_runs as rr
|
||||
|
||||
seen_register: set[str] = set()
|
||||
pos = 0
|
||||
while True:
|
||||
if report.exists():
|
||||
with open(report, "rb") as fh:
|
||||
fh.seek(pos)
|
||||
chunk = fh.read()
|
||||
pos = fh.tell()
|
||||
for raw in chunk.split(b"\n"):
|
||||
if not raw.strip():
|
||||
continue
|
||||
try:
|
||||
entry = json.loads(raw)
|
||||
except ValueError:
|
||||
continue
|
||||
await self._handle_entry(run_id, entry, seen_register, rr)
|
||||
await asyncio.sleep(0.3)
|
||||
|
||||
async def _handle_entry(self, run_id, entry, seen_register, rr) -> None:
|
||||
rtype = entry.get("$report_type")
|
||||
if rtype == "CollectReport":
|
||||
for item in entry.get("result", []):
|
||||
nodeid = item.get("nodeid")
|
||||
if not nodeid or "::" not in nodeid or nodeid in seen_register:
|
||||
continue
|
||||
seen_register.add(nodeid)
|
||||
path, name = _split_nodeid(nodeid)
|
||||
await self.hub.publish(
|
||||
"test.progress",
|
||||
{
|
||||
"type": "register",
|
||||
"nodeid": nodeid,
|
||||
"tier": tier_for(nodeid),
|
||||
"file": path,
|
||||
"testname": name,
|
||||
},
|
||||
)
|
||||
elif rtype == "TestReport":
|
||||
nodeid = entry.get("nodeid")
|
||||
when = entry.get("when")
|
||||
outcome = entry.get("outcome")
|
||||
if when == "setup":
|
||||
await self.hub.publish(
|
||||
"test.progress", {"type": "running", "nodeid": nodeid}
|
||||
)
|
||||
# Final outcome: the call phase normally, or a non-passed setup
|
||||
# (skip/error) that short-circuits the test.
|
||||
final = when == "call" or (when == "setup" and outcome != "passed")
|
||||
if final:
|
||||
duration = entry.get("duration")
|
||||
await self.hub.publish(
|
||||
"test.progress",
|
||||
{
|
||||
"type": "outcome",
|
||||
"nodeid": nodeid,
|
||||
"outcome": outcome,
|
||||
"duration": duration,
|
||||
},
|
||||
)
|
||||
longrepr = entry.get("longrepr")
|
||||
await rr.add_result(
|
||||
self.db,
|
||||
run_id,
|
||||
nodeid=nodeid,
|
||||
tier=tier_for(nodeid or ""),
|
||||
outcome=outcome,
|
||||
duration_s=duration,
|
||||
device_serial=None,
|
||||
longrepr=str(longrepr) if longrepr else None,
|
||||
)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user