Fix Property Updates, Add Timestamps, Unused Status Handling

.gitignore

@@ -63,4 +63,4 @@ target/
 .AppleDouble
 
 # VSCode Configs
-.vscode/
+.vscode/

CHANGELOG.md

@@ -1,9 +1,15 @@
 ## CHANGELOG
 
-### [v2.0.3] - Minor Updates, Connection Events and Timestamps
+### [v2.0.3] - Fix Property Updates, Add Timestamps, Unused Status Handling
+
+#### Changed / Fixed
+
+- Changed the default Status Property (`ST`) unit of measurement (UOM) to `ISY_PROP_NOT_SET = "-1"`: Some NodeServer and Z-Wave nodes do not make use of the `ST` (or status) property in the ISY and only report `aux_properties`; in addition, most NodeServer nodes do not report the `ST` property when all nodes are retrieved, they only report it when queried directly or in the Event Stream. Previously, there was no way to differentiate between Insteon Nodes that don't have a valid status yet (after ISY reboot) and the other types of nodes that don't report the property correctly since they both reported `ISY_VALUE_UNKNOWN`.  The `ISY_PROP_NOT_SET` allows differentiation between the two conditions based on having a valid UOM or not. Fixes #98.
+- Rewrite the Node status update receiver: currently, when a Node's status is updated, the `formatted` property is not updated and the `uom`/`prec` are updated with separate functions from outside of the Node's class. This updates the receiver to pass a `NodeProperty` instance into the Node, and allows the Node to update all of it's properties if they've changed, before reporting the status change to the subscribers. This makes the `formatted` property actually useful.
 
 #### Added
 
+- Added `*.last_update` and `*.last_changed` properties which are UTC Datetime Timestamps, to allow detection of stale data. Fixes #99
 - Add connection events for the Event Stream to allow subscription and callbacks. Attach a callback with `isy.connection_events(callback)` and receive a string with the event detail. See `constants.py` for events starting with prefix `ES_`.
 
 ### [v2.0.2] - Version 2.0 Initial Release

examples/connection_test.py

@@ -8,8 +8,6 @@
 from pyisy import ISY
 from pyisy.connection import Connection
 
-_LOGGER = logging.getLogger(__name__)
-
 # CONFIGURATION OPTIONS:
 ADDRESS = "my.isy.io"  # IP address or hostname of your ISY
 PORT = 443  # Port number to use for connection
@@ -19,12 +17,16 @@
 TLS_VER = 1.1  # TLS Version: 1.1 or 1.2, 0 for HTTP
 WEBROOT = ""  # Optional, for Advanced ISY Portal use
 
+LOG_LEVEL = logging.DEBUG
+# LOG_LEVEL = 5  # (Verbose) Use this level for printing event stream socket messages
+LOG_FORMAT = "%(asctime)s %(levelname)s [%(name)s] %(message)s"
+LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
+
+_LOGGER = logging.getLogger(__name__)
 
 def main(arguments):
     """Execute primary loop."""
-    fmt = "%(asctime)s %(levelname)s [%(name)s] %(message)s"
-    datefmt = "%Y-%m-%d %H:%M:%S"
-    logging.basicConfig(format=fmt, datefmt=datefmt, level=logging.DEBUG)
+    logging.basicConfig(format=LOG_FORMAT, datefmt=LOG_DATE_FORMAT, level=LOG_LEVEL)
 
     # Test the connection to ISY controller.
     try:
@@ -62,17 +64,20 @@ def main(arguments):
 
     _LOGGER.info("ISY connected: %s", isy.connected)
 
-    # Print a representation of all the Nodes
-    _LOGGER.debug(repr(isy.nodes))
-
-    if isy.connected:
-        # Connect to the Event Stream and print events in the Debug Log.
-        isy.auto_update = True
-    else:
+    if not isy.connected:
         _LOGGER.error(
             "Failed to connect to the ISY, please adjust settings and try again."
         )
         return
+    # Print a representation of all the Nodes
+    _LOGGER.debug(repr(isy.nodes))
+
+    # Get the rest of the detailed status information from the ISY
+    # not originally reported. This includes statuses for all NodeServer nodes.
+    isy.nodes.update()
+
+    # Connect to the Event Stream and print events in the Debug Log.
+    isy.auto_update = True
 
     try:
         while True:

pyisy/connection.py

@@ -23,6 +23,7 @@
     URL_PING,
     URL_PROGRAMS,
     URL_RESOURCES,
+    URL_STATUS,
     URL_SUBFOLDERS,
     URL_VARIABLES,
     VAR_INTEGER,
@@ -197,6 +198,13 @@ def get_nodes(self):
         req_url = self.compile_url([URL_NODES], {URL_MEMBERS: XML_FALSE})
         result = self.request(req_url)
         return result
+        return result
+
+    def get_status(self):
+        """Fetch the status of nodes/groups/scenes from the ISY."""
+        req_url = self.compile_url([URL_STATUS])
+        result = self.request(req_url)
+        return result
 
     def get_variable_defs(self):
         """Fetch the list of variables from the ISY."""

pyisy/constants.py

@@ -34,6 +34,7 @@
 ES_RECONNECTING = "reconnecting"
 
 ISY_VALUE_UNKNOWN = -1 * float("inf")
+ISY_PROP_NOT_SET = "-1"
 
 XML_ERRORS = (AttributeError, KeyError, ValueError, TypeError, IndexError, ExpatError)
 XML_PARSE_ERROR = "ISY Could not parse response, poorly formatted XML."
@@ -49,6 +50,8 @@
 ATTR_ID = "id"
 ATTR_INIT = "init"
 ATTR_INSTANCE = "instance"
+ATTR_LAST_CHANGED = "last_changed"
+ATTR_LAST_UPDATE = "last_update"
 ATTR_NODE_DEF_ID = "nodeDefId"
 ATTR_PARENT = "parentId"
 ATTR_PRECISION = "prec"
@@ -332,9 +335,11 @@
 UOM_FAN_MODES = "99"
 UOM_CLIMATE_MODES = "98"
 UOM_CLIMATE_MODES_ZWAVE = "67"
+UOM_DOUBLE_TEMP = "101"
 
 UOM_FRIENDLY_NAME = {
     "1": "A",
+    "2": "",  # Binary / On-Off
     "3": "btu/h",
     "4": "°C",
     "5": "cm",
@@ -413,6 +418,7 @@
     "90": "Hz",
     "91": "°",
     "92": "° South",
+    "100": "",
     "101": "° (x2)",
     "102": "kWs",
     "103": "$",
@@ -428,8 +434,8 @@
     "113": "",  # raw 3-byte signed value
     "114": "",  # raw 4-byte signed value
     "116": "mi",
-    "117": "mb",
-    "118": "hpa",
+    "117": "mbar",
+    "118": "hPa",
     "119": "Wh",
     "120": "in/day",
 }

pyisy/events.py

@@ -1,5 +1,4 @@
 """ISY Event Stream."""
-import datetime
 import socket
 import ssl
 from threading import Thread, ThreadError
@@ -26,7 +25,7 @@
     VERBOSE,
 )
 from .eventreader import ISYEventReader, ISYMaxConnections, ISYStreamDataError
-from .helpers import attr_from_xml, value_from_xml
+from .helpers import attr_from_xml, now, value_from_xml
 
 
 class EventStream:
@@ -95,7 +94,7 @@ def _route_message(self, msg):
             elif self._loaded == ES_INITIALIZING:
                 self._loaded = ES_LOADED
                 self.isy.connection_events.notify(ES_LOADED)
-            self._lasthb = datetime.datetime.now()
+            self._lasthb = now()
             self._hbwait = int(value_from_xml(xmldoc, ATTR_ACTION))
             self.isy.log.debug("ISY HEARTBEAT: %s", self._lasthb.isoformat())
         elif cntrl == PROP_STATUS:  # NODE UPDATE
@@ -207,7 +206,7 @@ def connected(self):
     def heartbeat_time(self):
         """Return the last ISY Heartbeat time."""
         if self._lasthb is not None:
-            return (datetime.datetime.now() - self._lasthb).seconds
+            return (now() - self._lasthb).seconds
         return 0.0
 
     def _lost_connection(self, delay=0):

pyisy/helpers.py

@@ -11,6 +11,7 @@
     ATTR_VALUE,
     INSTEON_RAMP_RATES,
     ISY_EPOCH_OFFSET,
+    ISY_PROP_NOT_SET,
     ISY_VALUE_UNKNOWN,
     PROP_BATTERY_LEVEL,
     PROP_RAMP_RATE,
@@ -37,7 +38,7 @@ def parse_xml_properties(xmldoc):
     """
     aux_props = {}
     state_set = False
-    state = NodeProperty(PROP_STATUS)
+    state = NodeProperty(PROP_STATUS, uom=ISY_PROP_NOT_SET)
 
     props = xmldoc.getElementsByTagName(TAG_PROPERTY)
     if not props:
@@ -143,6 +144,17 @@ def ntp_to_system_time(timestamp):
     return datetime.datetime.fromtimestamp(timestamp - ntp_delta)
 
 
+def now():
+    """Get the current system time.
+
+    Note: this module uses naive datetimes because the
+    ISY is highly inconsistent with time conventions
+    and does not present enough information to accurately
+    mangage DST without significant guessing and effort.
+    """
+    return datetime.datetime.now()
+
+
 class EventEmitter:
     """Event Emitter class."""
 
@@ -183,7 +195,13 @@ class NodeProperty(dict):
     """Class to hold result of a control event or node aux property."""
 
     def __init__(
-        self, control, value=ISY_VALUE_UNKNOWN, prec="0", uom="", formatted=None
+        self,
+        control,
+        value=ISY_VALUE_UNKNOWN,
+        prec="0",
+        uom="",
+        formatted=None,
+        address=None,
     ):
         """Initialize an control result or aux property."""
         super().__init__(
@@ -193,8 +211,14 @@ def __init__(
             prec=prec,
             uom=uom,
             formatted=(formatted if formatted is not None else value),
+            address=address,
         )
 
+    @property
+    def address(self):
+        """Report the address of the node with this property."""
+        return self["address"]
+
     @property
     def control(self):
         """Report the event control string."""
@@ -223,8 +247,9 @@ def formatted(self):
     def __str__(self):
         """Return just the event title to prevent breaking changes."""
         return (
-            f"NodeProperty('{self.control}': value='{self.value}' "
-            f"prec='{self.prec}' uom='{self.uom}' formatted='{self.formatted}')"
+            f"NodeProperty('{self.address}': control='{self.control}', "
+            f"value='{self.value}', prec='{self.prec}', "
+            f"uom='{self.uom}', formatted='{self.formatted}')"
         )
 
     __repr__ = __str__