Implement new zigpy network state (#445)

* [WIP] Implement new zigpy network state

* [WIP] Implement writing network settings

* Set NWK and APS frame counters

* Change key table size only if the new keys don't fit

* Style cleanup

* Directly create `EZSPCoordinator` instead of using a quirk

* Use zigpy types when creating network state objects

* Allow `load_network_info` to be called before/after `start_network`

* Allow `node_info.ieee` to be `None`

* Use `t.EUI64.UNKNOWN` instead of `None` to represent an unset IEEE addr

* Do not leak EZSP types into zigpy

* Fix network formation when a hashed TCLK is generated

* Joins were not being properly permitting after fcf0b49

* Use the new zigpy `add_endpoint` method

* Call `register_endpoints` when connecting

* Always leave the current network during formation

* Get some unit tests passing

* Add unit tests for `bellows.zigbee.util`

* Increase `bellows.uart` test coverage to 100%

* Ignore `.DS_Store` files

* Use new `pytest-asyncio` behavior

* Newer versions of EmberZNet use a different response for `getChildData`

* Use zigpy data types when populating `state`

* Correct the TCLK's `partner_ieee` to use the node's IEEE address

* Unit test `load_network_info`

Foo

* Assume the TCLK is always the well-known key

* Add a few more unknown bitfields

* Refactor code dealing with keys and add some comments

* Write initial unit tests for `write_network_info`

* Remove ignored flake8 errors from unit tests

* Only use one pair of `network_info` and `node_info` fixtures

* Test new code in `bellows.zigbee.util`

* Add unit test for when `getMfgToken` is missing

* Unit test `_ensure_network_running`

* Fix startup delay caused by slow coordinator initialization

* Key table size can't be adjusted

* Unit test `getChildData` for EZSP v6

* Unit test `leaveNetwork` failing with `INVALID_CALL`

* Add the coordinator to the zigpy device dictionary before initialization

* Do not back up the address tables on EZSPv4, it is unstable

* Do not set frame counters when using EZSPv4

* Fix UART port close on RSTACK message during startup

When a RSTACK message is processed right after the UART has been opened,
it causes EZSP.enter_failed_state() getting called at a point where the
application callbacks are not registered yet. In that case the UART
will get closed and it won't get opened again. Bellows is stuck with a
closed transport.

Avoid this issue by not closing the port in case there is no application
callback registered yet.

Typically, it is unlikely that a RSTACK message arrives right when
the port gets opened (the race window is very narrow). However, with
hardware flow control opening the port leads to RTS signal to get
asserted which causes the radio to send pending messages, e.g. resets
caused by EmberZNet watchdog.

Note: With hardware flow control this is only the case if the tty "hupcl"
option is set. The option is set by default, but cleared by tools like
GNU screen. This option makes sure that the RTS signal is deasserted
while the port is closed. Pyserial/bellows does not change the state
of that option.

* Unit test EZSP v4 changes

* Fix tests

* Include radio library metadata in network info

* Fix unit tests broken by new `metadata` key

* Do not set the `HAVE_TRUST_CENTER_EUI64` bit when forming a network

* Bump minimum required zigpy version to 0.47.0

* Fix unit tests for Python 3.7

Co-authored-by: Stefan Agner <[email protected]>

Author: puddly

.gitignore

@@ -70,4 +70,8 @@ ENV/
 .*.swp
 
 # Visual Studio Code
-.vscode
+.vscode
+
+
+# macOS
+.DS_Store

bellows/ezsp/v7/commands.py

@@ -137,7 +137,7 @@
     "getChildData": (
         0x4A,
         (t.uint8_t,),
-        (t.EmberStatus, t.EmberNodeId, t.EmberEUI64, t.EmberNodeType),
+        (t.EmberStatus, t.EmberChildData),
     ),
     "getSourceRouteTableTotalSize": (0xC3, (), (t.uint8_t,)),
     "getSourceRouteTableFilledSize": (0xC2, (), (t.uint8_t,)),

bellows/ezsp/v7/types/struct.py

@@ -189,3 +189,25 @@ class EmberPerDeviceDutyCycle(EzspStruct):
     nodeId: named.EmberNodeId
     # Amount of overall duty cycle consumed (up to suspend limit).
     dutyCycleConsumed: named.EmberDutyCycleHectoPct
+
+
+class EmberChildData(EzspStruct):
+    """A structure containing a child node's data."""
+
+    # The EUI64 of the child
+    eui64: named.EmberEUI64
+    # The node type of the child
+    type: named.EmberNodeType
+    # The short address of the child
+    id: named.EmberNodeId
+    # The phy of the child
+    phy: basic.uint8_t
+    # The power of the child
+    power: basic.uint8_t
+    # The timeout of the child
+    timeout: basic.uint8_t
+
+    # The GPD's EUI64.
+    # gpdIeeeAddress: named.EmberEUI64
+    # The GPD's source ID.
+    # sourceId: basic.uint32_t

bellows/ezsp/v8/commands.py

@@ -134,7 +134,7 @@
     "getChildData": (
         0x004A,
         (t.uint8_t,),
-        (t.EmberStatus, t.EmberNodeId, t.EmberEUI64, t.EmberNodeType),
+        (t.EmberStatus, t.EmberChildData),
     ),
     "getSourceRouteTableTotalSize": (0x00C3, (), (t.uint8_t,)),
     "getSourceRouteTableFilledSize": (0x00C2, (), (t.uint8_t,)),

bellows/ezsp/v8/types/struct.py

@@ -202,3 +202,25 @@ class EmberTransientKeyData(EzspStruct):
     # The number of seconds remaining before the key is automatically timed out of the
     # transient key table.
     remainingTimeSeconds: basic.uint16_t
+
+
+class EmberChildData(EzspStruct):
+    """A structure containing a child node's data."""
+
+    # The EUI64 of the child
+    eui64: named.EmberEUI64
+    # The node type of the child
+    type: named.EmberNodeType
+    # The short address of the child
+    id: named.EmberNodeId
+    # The phy of the child
+    phy: basic.uint8_t
+    # The power of the child
+    power: basic.uint8_t
+    # The timeout of the child
+    timeout: basic.uint8_t
+
+    # The GPD's EUI64.
+    # gpdIeeeAddress: named.EmberEUI64
+    # The GPD's source ID.
+    # sourceId: basic.uint32_t

bellows/types/named.py

@@ -610,6 +610,9 @@ class EmberStatus(basic.enum8):
     # An index was passed into the function that was larger than the valid
     # range.
     INDEX_OUT_OF_RANGE = 0xB1
+    # The passed key data is not valid. A key of all zeros or all F's are reserved
+    # values and cannot be used.
+    KEY_INVALID = 0xB2
     # There are no empty entries left in the table.
     TABLE_FULL = 0xB4
     # The requested table entry has been erased and contains no valid data.
@@ -970,6 +973,8 @@ class EmberCurrentSecurityBitmask(basic.bitmap16):
     GLOBAL_LINK_KEY = 0x0004
     # This denotes that the node has a Trust Center Link Key.
     HAVE_TRUST_CENTER_LINK_KEY = 0x0010
+    # TODO: 0x0020 is unknown
+    # TODO: 0x0040 is unknown
     # This denotes that the Trust Center is using a Hashed Link Key.
     TRUST_CENTER_USES_HASHED_LINK_KEY = 0x0084
 
@@ -994,6 +999,12 @@ class EmberKeyStructBitmask(basic.bitmap16):
     # hears a device announce from the partner indicating it is not
     # an 'RX on when idle' device.
     KEY_PARTNER_IS_SLEEPY = 0x0020
+    # This indicates that the transient key which is being added is unconfirmed. This
+    # bit is set when we add a transient key while the EmberTcLinkKeyRequestPolicy is
+    # EMBER_ALLOW_TC_LINK_KEY_REQUEST_AND_GENERATE_NEW_KEY
+    UNCONFIRMED_TRANSIENT_KEY = 0x0040
+
+    # TODO: 0x0080 is unknown
 
 
 class EmberKeyStatus(basic.enum8):
@@ -1757,3 +1768,22 @@ class sl_Status(basic.enum32):
     SL_STATUS_WIFI_RETRY_EXCEEDED = 0x0B1F
     #  The request failed because the MSDU life time was exceeded
     SL_STATUS_WIFI_TX_LIFETIME_EXCEEDED = 0x0B20
+
+
+class EmberDistinguishedNodeId(basic.enum16):
+    """A distinguished network ID that will never be assigned to any node"""
+
+    # This value is used when getting the remote node ID from the address or binding
+    # tables. It indicates that the address or binding table entry is currently in use
+    # and network address discovery is underway.
+    DISCOVERY_ACTIVE = 0xFFFC
+
+    # This value is used when getting the remote node ID from the address or binding
+    # tables. It indicates that the address or binding table entry is currently in use
+    # but the node ID corresponding to the EUI64 in the table is currently unknown.
+    UNKNOWN = 0xFFFD
+
+    # This value is used when setting or getting the remote node ID in the address table
+    # or getting the remote node ID from the binding table. It indicates that the
+    # address or binding table entry is not in use.
+    TABLE_ENTRY_UNUSED = 0xFFFF

bellows/types/struct.py

@@ -2,8 +2,6 @@
 import inspect
 import typing
 
-import zigpy.state as app_state
-
 from . import basic, named
 
 NoneType = type(None)
@@ -282,19 +280,6 @@ class EmberNetworkParameters(EzspStruct):
     # method.
     channels: named.Channels
 
-    @property
-    def zigpy_network_information(self) -> app_state.NetworkInformation:
-        """Convert to NetworkInformation."""
-        r = app_state.NetworkInformation(
-            self.extendedPanId,
-            app_state.t.PanId(self.panId),
-            self.nwkUpdateId,
-            app_state.t.NWK(self.nwkManagerId),
-            self.radioChannel,
-            channel_mask=self.channels,
-        )
-        return r
-
 
 class EmberZigbeeNetwork(EzspStruct):
     # The parameters of a ZigBee network.