diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 0000000..ede1f6e --- /dev/null +++ b/bench/README.md @@ -0,0 +1,48 @@ +# Bench host setup + +Host-side pieces for the HIL bench (`tsi-ha`). Everything else in this repo runs inside +the `firmware-tester` container; these do not, because they drive hardware the container +has no access to. + +## BLE flood + +`scripts/ble_flood.py` advertises from a fresh random address ~40 times a second, so each +rotation costs a node a new fingerprint slot. It exists to make the slow heap decline in +[ESPresense#2309](https://github.com/ESPresense/ESPresense/issues/2309) show up in a HIL +window instead of over days on someone's shelf. + +It talks to the adapter over `HCI_CHANNEL_USER`, so **BlueZ is not required and should not +be installed** — the kernel hands over the controller exclusively and `bluetoothd` would +only fight for it. + +Requires a USB Bluetooth adapter passed through to the VM. Verify the kernel sees it: + +```bash +ls /sys/class/bluetooth # expect hci0 +python3 ble_flood.py --selftest # framing + address rules, no hardware needed +sudo python3 ble_flood.py --index 0 --seconds 30 # 30s live burst +``` + +Install: + +```bash +sudo install -m 755 scripts/ble_flood.py /usr/local/bin/ble_flood.py +sudo install -m 644 bench/ble-flood.service /etc/systemd/system/ +sudo systemctl enable --now ble-flood +``` + +The service idles until a request file appears in `/var/lock/woodpecker/bleflood.d/`. Each +HIL step creates one on entry and removes it on exit, so the flood only runs during tests — +otherwise every ESPresense node within range spends the day logging junk MACs. It is a +directory, not a single flag, because the four device steps run in parallel and the first +one to finish must not cut the flood out from under the rest. Request files older than +`--max-age` (9h) are ignored, so a hard-killed container cannot leave the bench advertising +forever. + +To flood by hand (e.g. reproducing a report): + +```bash +sudo mkdir -p /var/lock/woodpecker/bleflood.d +sudo touch /var/lock/woodpecker/bleflood.d/manual # journalctl -fu ble-flood to watch +sudo rm /var/lock/woodpecker/bleflood.d/manual +``` diff --git a/bench/ble-flood.service b/bench/ble-flood.service new file mode 100644 index 0000000..52892b5 --- /dev/null +++ b/bench/ble-flood.service @@ -0,0 +1,24 @@ +[Unit] +Description=BLE advertisement flood for the ESPresense HIL bench +Documentation=https://github.com/ESPresense/firmware-tester +After=network.target + +[Service] +# Runs continuously but only advertises while a HIL step asks for it. Steps already +# bind-mount /var/lock/woodpecker, so each one drops a request file there and removes it on +# exit — no host access, no extra plumbing, and the bench is quiet between runs instead of +# filling every ESPresense node in the house with junk fingerprints. +ExecStart=/usr/local/bin/ble_flood.py --index 0 --rate 40 --flag-dir /var/lock/woodpecker/bleflood.d +Restart=always +RestartSec=5 +# HCI_CHANNEL_USER needs CAP_NET_ADMIN and an adapter no one else has powered up. +AmbientCapabilities=CAP_NET_ADMIN +CapabilityBoundingSet=CAP_NET_ADMIN +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +ReadWritePaths=/var/lock/woodpecker + +[Install] +WantedBy=multi-user.target diff --git a/scripts/ble_flood.py b/scripts/ble_flood.py new file mode 100644 index 0000000..820a2a8 --- /dev/null +++ b/scripts/ble_flood.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 +"""Flood the bench with BLE advertisements from a fresh random address every time. + +Why: ESPresense fingerprints a *static random* address by MAC (BleFingerprint.cpp, +ID_TYPE_RAND_STATIC_MAC when the top two bits of the MSB are set), so every rotation +here costs the node a new fingerprint slot against a pool of 100-200. That is the +churn a real room full of phones produces, compressed — the load under which +ESPresense#2309's slow heap decline shows up in hours instead of days. + +The adapter is driven raw over HCI_CHANNEL_USER, which means BlueZ is not involved and +does not need to be installed: the kernel hands us the controller exclusively and +nothing else can fight us for it. Requires root (CAP_NET_ADMIN) and an adapter that no +other process has powered up. + + ble_flood.py --index 0 --rate 40 + ble_flood.py --index 0 --flag /var/lock/woodpecker/bleflood.on # only while flag exists + ble_flood.py --selftest # no hardware needed +""" + +import argparse +import ctypes +import fcntl +import os +import secrets +import select +import socket +import struct +import time + +AF_BLUETOOTH = 31 +BTPROTO_HCI = 1 +HCI_CHANNEL_USER = 1 +HCIDEVDOWN = 0x400448CA + +HCI_COMMAND_PKT = 0x01 +HCI_EVENT_PKT = 0x04 +EVT_CMD_COMPLETE = 0x0E +EVT_CMD_STATUS = 0x0F + +OCF_RESET = 0x0C03 +OCF_LE_SET_RANDOM_ADDRESS = 0x2005 +OCF_LE_SET_ADV_PARAMETERS = 0x2006 +OCF_LE_SET_ADV_DATA = 0x2008 +OCF_LE_SET_ADV_ENABLE = 0x200A + +ADV_NONCONN_IND = 0x03 +OWN_ADDR_TYPE_RANDOM = 0x01 +ADV_INTERVAL = 0x0020 # 20 ms — the BLE minimum, so a packet lands on all 3 channels + + +def hci_command(opcode, params=b""): + """Frame one HCI command packet: type, opcode (LE), parameter length, parameters.""" + if len(params) > 255: + raise ValueError(f"HCI parameters too long: {len(params)}") + return struct.pack(" 31: + raise ValueError(f"advertising payload too long: {len(fields)}") + return bytes([len(fields)]) + fields.ljust(31, b"\x00") + + +def adv_parameters(): + return struct.pack( + " 6 else 0x00 + if pkt[1] == EVT_CMD_STATUS: # type, 0x0f, plen, status, ncmd, opcode(2) + if len(pkt) < 7 or struct.unpack_from(" re-address -> enable. + """ + send(sock, OCF_RESET) + time.sleep(0.1) + send(sock, OCF_LE_SET_ADV_PARAMETERS, adv_parameters()) + + interval = 1.0 / rate + started = time.monotonic() + rotations = 0 + reported = started + advertising = False + + while True: + if stop_after and time.monotonic() - started >= stop_after: + break + if not flood_requested(flag_dir, max_age): + if advertising: + send(sock, OCF_LE_SET_ADV_ENABLE, b"\x00") + advertising = False + print(f"[flood] paused — no requests in {flag_dir}", flush=True) + time.sleep(2) + continue + if not advertising and flag_dir: + print(f"[flood] resumed — request present in {flag_dir}", flush=True) + + cycle = time.monotonic() + address = random_static_address() + send(sock, OCF_LE_SET_ADV_ENABLE, b"\x00") + send(sock, OCF_LE_SET_RANDOM_ADDRESS, address) + send(sock, OCF_LE_SET_ADV_DATA, advertising_payload(address)) + send(sock, OCF_LE_SET_ADV_ENABLE, b"\x01") + advertising = True + rotations += 1 + + now = time.monotonic() + if now - reported >= 30: + print(f"[flood] {rotations} unique addresses in {now - started:.0f}s " + f"({rotations / (now - started):.1f}/s)", flush=True) + reported = now + time.sleep(max(0.0, interval - (now - cycle))) + + send(sock, OCF_LE_SET_ADV_ENABLE, b"\x00") + print(f"[flood] stopped after {rotations} unique addresses", flush=True) + + +def selftest(): + """Check the packet framing and address rules without touching an adapter.""" + pkt = hci_command(OCF_LE_SET_ADV_ENABLE, b"\x01") + assert pkt == b"\x01\x0a\x20\x01\x01", pkt.hex() + + assert hci_command(OCF_RESET) == b"\x01\x03\x0c\x00" + + for _ in range(2000): + addr = random_static_address() + assert len(addr) == 6 + # The MSB is the last byte on the wire; ESPresense keys ID_TYPE_RAND_STATIC_MAC + # off exactly this test, so if it ever fails the flood stops being fingerprinted. + assert addr[5] & 0xC0 == 0xC0, addr.hex() + rest = int.from_bytes(addr, "little") & ((1 << 46) - 1) + assert rest not in (0, (1 << 46) - 1) + + assert len({random_static_address() for _ in range(5000)}) == 5000, "addresses repeat" + + # Every advert must carry an identity unique to its address, or ESPresense merges them. + addr_a, addr_b = random_static_address(), random_static_address() + payload = advertising_payload(addr_a) + assert len(payload) == 32, len(payload) + assert payload[0] == 21 and payload[1:4] == b"\x02\x01\x06", payload.hex() + assert payload[4:6] == b"\x11\x09", payload.hex() # 16-byte name, "complete local name" + assert payload[6:22] == b"HIL-" + addr_a[::-1].hex().encode(), payload.hex() + assert advertising_payload(addr_b) != payload, "payload must vary with the address" + assert len({advertising_payload(random_static_address()) for _ in range(2000)}) == 2000 + assert len(adv_parameters()) == 15, len(adv_parameters()) + + try: + hci_command(OCF_LE_SET_ADV_DATA, b"\x00" * 256) + raise AssertionError("oversized parameters must be rejected") + except ValueError: + pass + + # Completion parsing decides whether a rejected command is noticed at all. If this + # goes wrong the flood happily reports thousands of addresses while radiating none. + op = OCF_LE_SET_ADV_ENABLE + complete_ok = bytes([HCI_EVENT_PKT, EVT_CMD_COMPLETE, 4, 1]) + struct.pack("{free1:.0f}, maxHeap {max0:.0f}->{max1:.0f}, " + f"fingerprints {fp0}->{fp1}, over {format_duration(int(span))}") + + if free1 < free0 * (1 - HEAP_DECLINE_FRAC): + lost = (1 - free1 / free0) * 100 + return f"Free heap fell {lost:.0f}% ({summary})" + print(f"[hil] heap trend OK ({summary})", flush=True) + return None + + class _Bug(Exception): """A /json contract violation that must fail the build.""" @@ -224,6 +322,9 @@ def main(): booted = False saw_scan_result = False json_failure = [] # appended to by the /json check thread + heap_samples = [] # appended to by the heap sampler thread + heap_problems = [] # ditto, for "the check could not run" conditions + stop_sampling = threading.Event() try: while True: @@ -242,6 +343,11 @@ def main(): f"{format_duration(args.duration)}." ) sys.exit(4) + stop_sampling.set() + decline = heap_verdict(heap_samples, elapsed, heap_problems) + if decline: + print(f"FAIL: {decline}") + sys.exit(8) print(f"PASS: {format_duration(args.duration)} elapsed cleanly.") sys.exit(0) @@ -269,15 +375,29 @@ def main(): print(f"[hil] Boot confirmed at {elapsed:.1f}s") match = IP_PATTERN.search(line) if match: - # Background thread so the serial buffer keeps draining while we probe. + # Background threads so the serial buffer keeps draining while we probe. threading.Thread( target=json_endpoint_check, args=(match.group(1), json_failure), daemon=True, ).start() + threading.Thread( + target=heap_sampler, + args=(match.group(1), heap_samples, stop_sampling, heap_problems), + daemon=True, + ).start() else: print(f"[hil] /json check SKIPPED — no IP in {line!r}") + # A restart after boot resets every other signal here — fresh boot, fresh heap, + # fresh scan results — so without this the window passes while the node is + # actually power-cycling every few hours (ESPresense#2309 on the S3). + elif booted and (REBOOT_PATTERN in line or OOM_RESTART_PATTERN in line): + why = ("firmware low-heap watchdog fired" if OOM_RESTART_PATTERN in line + else "unexpected restart") + print(f"FAIL: Node restarted {elapsed:.0f}s into the run — {why}: {line.strip()!r}") + sys.exit(7) + if MQTT_RECONNECT_LIMIT_PATTERN in line: print(f"FAIL: Firmware MQTT reconnect limit reached: '{MQTT_RECONNECT_LIMIT_PATTERN}'") sys.exit(5) diff --git a/scripts/test_heap_trend.py b/scripts/test_heap_trend.py new file mode 100644 index 0000000..23c620d --- /dev/null +++ b/scripts/test_heap_trend.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Prove heap_verdict fails on a real decline and stays quiet on healthy noise. + +The shapes come from ESPresense#2309: the S3 slid 85KB -> 33KB over ~6h with the +fingerprint count flat, while the C3 sat at ~110KB for the same window. One of those +must fail the build and the other must not. +""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from hil_monitor import HEAP_TREND_MIN_SECS, heap_verdict # noqa: E402 + +HOURS_6 = 6 * 3600 + + +def samples(values, fingerprints=29): + return [(i * 60.0, v, v // 3, fingerprints) for i, v in enumerate(values)] + + +def ramp(start, end, n): + step = (end - start) / (n - 1) + return [int(start + step * i) for i in range(n)] + + +def wobble(level, n): + """Healthy heap is noisy, not flat — alternate around the level.""" + return [level + (2000 if i % 2 else -2000) for i in range(n)] + + +# The S3: a steady slide with the fingerprint count flat. Must fail. +verdict = heap_verdict(samples(ramp(85000, 33000, 360)), HOURS_6) +assert verdict and "fell" in verdict, f"leak not caught: {verdict!r}" +assert "fingerprints 29->29" in verdict, verdict +print(f"leak -> {verdict}") + +# The C3: noisy but level. Must pass. +verdict = heap_verdict(samples(wobble(110000, 360)), HOURS_6) +assert verdict is None, f"healthy node failed: {verdict!r}" +print("healthy -> pass") + +# A dip that recovers is not a leak — the tail is what matters, and the median at each +# edge keeps a single ugly sample from deciding the build. +dip = wobble(110000, 150) + ramp(110000, 60000, 60) + ramp(60000, 108000, 150) +verdict = heap_verdict(samples(dip), HOURS_6) +assert verdict is None, f"transient dip failed the build: {verdict!r}" +print("dip -> pass") + +# A 25% floor means a shallow slide is tolerated; anything past it is not. +assert heap_verdict(samples(ramp(100000, 80000, 360)), HOURS_6) is None, "20% should pass" +assert heap_verdict(samples(ramp(100000, 70000, 360)), HOURS_6), "30% should fail" +print("threshold -> 20% passes, 30% fails") + +# A 3-minute PR run cannot say anything about a multi-hour slope, so it must not try. +assert heap_verdict(samples(ramp(85000, 33000, 360)), 180) is None, "short run must not judge" +assert heap_verdict(samples(ramp(85000, 33000, 8)), HOURS_6) is None, "too few samples to judge" +print("short run -> skipped") + +# Firmware without /json/tele must fail loudly. A heap check that silently samples nothing +# and still reports PASS is how #2309 survived an 8h soak in the first place. +verdict = heap_verdict([], HOURS_6, ["missing"]) +assert verdict and "missing" in verdict, f"old firmware not flagged: {verdict!r}" +print(f"no endpoint -> {verdict}") + +# But a node the runner simply cannot reach is not a firmware fault — still a skip. +assert heap_verdict([], HOURS_6) is None, "unreachable node must not fail the build" +assert heap_verdict([], 180, ["missing"]) is None, "short run still must not judge" +print("unreachable -> skipped") + +print("all heap trend checks passed")