IOTData: Fix delta calculation (#7814)

moto/core/utils.py

@@ -337,6 +337,8 @@ def merge_dicts(
                 merge_dicts(dict1[key], dict2[key], remove_nulls)
             else:
                 dict1[key] = dict2[key]
+                if isinstance(dict1[key], dict):
+                    remove_null_from_dict(dict1)
             if dict1[key] == {} and remove_nulls:
                 dict1.pop(key)
         else:
@@ -345,6 +347,14 @@ def merge_dicts(
                 dict1.pop(key)
 
 
+def remove_null_from_dict(dct: Dict[str, Any]) -> None:
+    for key in list(dct.keys()):
+        if dct[key] is None:
+            dct.pop(key)
+        elif isinstance(dct[key], dict):
+            remove_null_from_dict(dct[key])
+
+
 def aws_api_matches(pattern: str, string: Any) -> bool:
     """
     AWS API can match a value based on a glob, or an exact match

moto/iotdata/models.py

@@ -72,12 +72,17 @@ def create_from_previous_version(  # type: ignore[misc]
 
     @classmethod
     def parse_payload(cls, desired: Optional[str], reported: Optional[str]) -> Any:  # type: ignore[misc]
-        if desired is None:
-            delta = reported
-        elif reported is None:
+        if not desired and not reported:
+            delta = None
+        elif reported is None and desired:
             delta = desired
+        elif desired and reported:
+            delta = jsondiff.diff(reported, desired)
+            delta.pop(jsondiff.add, None)  # type: ignore
+            delta.pop(jsondiff.delete, None)  # type: ignore
+            delta.pop(jsondiff.replace, None)  # type: ignore
         else:
-            delta = jsondiff.diff(desired, reported)
+            delta = None
         return delta
 
     def _create_metadata_from_state(self, state: Any, ts: Any) -> Any:
@@ -129,11 +134,11 @@ def to_dict(self, include_delta: bool = True) -> Dict[str, Any]:
             return {"timestamp": self.timestamp, "version": self.version}
         delta = self.parse_payload(self.desired, self.reported)
         payload = {}
-        if self.desired is not None:
+        if self.desired:
             payload["desired"] = self.desired
-        if self.reported is not None:
+        if self.reported:
             payload["reported"] = self.reported
-        if include_delta and (delta is not None and len(delta.keys()) != 0):
+        if include_delta and delta:
             payload["delta"] = delta
 
         metadata = {}
@@ -214,11 +219,9 @@ def delete_thing_shadow(
     def publish(self, topic: str, payload: bytes) -> None:
         self.published_payloads.append((topic, payload))
 
-    def list_named_shadows_for_thing(self, thing_name: str) -> List[FakeShadow]:
+    def list_named_shadows_for_thing(self, thing_name: str) -> List[str]:
         thing = self.iot_backend.describe_thing(thing_name)
-        return [
-            shadow for name, shadow in thing.thing_shadows.items() if name is not None
-        ]
+        return [name for name in thing.thing_shadows.keys() if name is not None]
 
 
 iotdata_backends = BackendDict(IoTDataPlaneBackend, "iot-data")

moto/iotdata/responses.py

@@ -63,4 +63,4 @@ def publish(self) -> str:
     def list_named_shadows_for_thing(self) -> str:
         thing_name = self._get_param("thingName")
         shadows = self.iotdata_backend.list_named_shadows_for_thing(thing_name)
-        return json.dumps({"results": [shadow.to_dict() for shadow in shadows]})
+        return json.dumps({"results": shadows})

tests/test_iotdata/__init__.py

@@ -1 +1,56 @@
-# This file is intentionally left blank.
+import os
+from functools import wraps
+from typing import TYPE_CHECKING, Callable, TypeVar
+from uuid import uuid4
+
+import boto3
+
+from moto import mock_aws
+
+if TYPE_CHECKING:
+    from typing_extensions import ParamSpec
+
+    P = ParamSpec("P")
+
+T = TypeVar("T")
+
+
+def iot_aws_verified() -> "Callable[[Callable[P, T]], Callable[P, T]]":
+    """
+    Function that is verified to work against AWS.
+    Can be run against AWS at any time by setting:
+      MOTO_TEST_ALLOW_AWS_REQUEST=true
+
+    If this environment variable is not set, the function runs in a `mock_aws` context.
+    """
+
+    def inner(func: "Callable[P, T]") -> "Callable[P, T]":
+        @wraps(func)
+        def pagination_wrapper(*args: "P.args", **kwargs: "P.kwargs") -> T:
+            allow_aws_request = (
+                os.environ.get("MOTO_TEST_ALLOW_AWS_REQUEST", "false").lower() == "true"
+            )
+
+            if allow_aws_request:
+                return _create_thing_and_execute_test(func, *args, **kwargs)
+            else:
+                with mock_aws():
+                    return _create_thing_and_execute_test(func, *args, **kwargs)
+
+        return pagination_wrapper
+
+    return inner
+
+
+def _create_thing_and_execute_test(
+    func: "Callable[P, T]", *args: "P.args", **kwargs: "P.kwargs"
+) -> T:
+    iot_client = boto3.client("iot", region_name="ap-northeast-1")
+    name = str(uuid4())
+
+    iot_client.create_thing(thingName=name)
+
+    try:
+        return func(*args, **kwargs, name=name)  # type: ignore
+    finally:
+        iot_client.delete_thing(thingName=name)

tests/test_iotdata/test_iotdata.py

@@ -1,5 +1,6 @@
 import json
 import sys
+from typing import Dict, Optional
 from unittest import SkipTest
 
 import boto3
@@ -11,16 +12,17 @@
 from moto.core import DEFAULT_ACCOUNT_ID as ACCOUNT_ID
 from moto.utilities.distutils_version import LooseVersion
 
+from . import iot_aws_verified
+
 boto3_version = sys.modules["botocore"].__version__
 
 
-@mock_aws
-def test_basic() -> None:
-    iot_client = boto3.client("iot", region_name="ap-northeast-1")
+@iot_aws_verified()
+@pytest.mark.aws_verified
+def test_basic(name: Optional[str] = None) -> None:
     client = boto3.client("iot-data", region_name="ap-northeast-1")
-    name = "my-thing"
+
     raw_payload = b'{"state": {"desired": {"led": "on"}}}'
-    iot_client.create_thing(thingName=name)
 
     with pytest.raises(ClientError):
         client.get_thing_shadow(thingName=name)
@@ -47,13 +49,11 @@ def test_basic() -> None:
         client.get_thing_shadow(thingName=name)
 
 
-@mock_aws
-def test_update() -> None:
-    iot_client = boto3.client("iot", region_name="ap-northeast-1")
+@iot_aws_verified()
+@pytest.mark.aws_verified
+def test_update(name: Optional[str] = None) -> None:
     client = boto3.client("iot-data", region_name="ap-northeast-1")
-    name = "my-thing"
     raw_payload = b'{"state": {"desired": {"led": "on"}}}'
-    iot_client.create_thing(thingName=name)
 
     # first update
     res = client.update_thing_shadow(thingName=name, payload=raw_payload)
@@ -97,14 +97,13 @@ def test_update() -> None:
     assert ex.value.response["Error"]["Message"] == "Version conflict"
 
 
-@mock_aws
-def test_create_named_shadows() -> None:
+@iot_aws_verified()
+@pytest.mark.aws_verified
+def test_create_named_shadows(name: Optional[str] = None) -> None:
     if LooseVersion(boto3_version) < LooseVersion("1.29.0"):
         raise SkipTest("Parameter only available in newer versions")
-    iot_client = boto3.client("iot", region_name="ap-northeast-1")
     client = boto3.client("iot-data", region_name="ap-northeast-1")
-    thing_name = "my-thing"
-    iot_client.create_thing(thingName=thing_name)
+    thing_name = name
 
     # default shadow
     default_payload = json.dumps({"state": {"desired": {"name": "default"}}})
@@ -128,23 +127,16 @@ def test_create_named_shadows() -> None:
     # List named shadows
     shadows = client.list_named_shadows_for_thing(thingName=thing_name)["results"]
     assert len(shadows) == 2
-
-    for shadow in shadows:
-        shadow.pop("metadata")
-        shadow.pop("timestamp")
-        shadow.pop("version")
-
-    # Verify both named shadows are present
-    for name in ["shadow1", "shadow2"]:
-        assert {
-            "state": {"reported": {"name": name}, "delta": {"name": name}}
-        } in shadows
+    assert "shadow1" in shadows
+    assert "shadow2" in shadows
 
     # Verify we can delete a named shadow
     client.delete_thing_shadow(thingName=thing_name, shadowName="shadow2")
 
-    with pytest.raises(ClientError):
-        client.get_thing_shadow(thingName="shadow1")
+    with pytest.raises(ClientError) as exc:
+        client.get_thing_shadow(thingName=thing_name, shadowName="shadow2")
+    err = exc.value.response["Error"]
+    assert err["Code"] == "ResourceNotFoundException"
 
     # The default and other named shadow are still there
     assert "payload" in client.get_thing_shadow(thingName=thing_name)
@@ -171,38 +163,86 @@ def test_publish() -> None:
         assert ("test/topic4", b"string") in mock_backend.published_payloads
 
 
-@mock_aws
-def test_delete_field_from_device_shadow() -> None:
-    test_thing_name = "TestThing"
-
-    iot_raw_client = boto3.client("iot", region_name="eu-central-1")
-    iot_raw_client.create_thing(thingName=test_thing_name)
-    iot = boto3.client("iot-data", region_name="eu-central-1")
+@iot_aws_verified()
+@pytest.mark.aws_verified
+def test_delete_field_from_device_shadow(name: Optional[str] = None) -> None:
+    iot = boto3.client("iot-data", region_name="ap-northeast-1")
 
     iot.update_thing_shadow(
-        thingName=test_thing_name,
+        thingName=name,
         payload=json.dumps({"state": {"desired": {"state1": 1, "state2": 2}}}),
     )
-    response = json.loads(
-        iot.get_thing_shadow(thingName=test_thing_name)["payload"].read()
-    )
+    response = json.loads(iot.get_thing_shadow(thingName=name)["payload"].read())
     assert len(response["state"]["desired"]) == 2
 
     iot.update_thing_shadow(
-        thingName=test_thing_name,
+        thingName=name,
         payload=json.dumps({"state": {"desired": {"state1": None}}}),
     )
-    response = json.loads(
-        iot.get_thing_shadow(thingName=test_thing_name)["payload"].read()
-    )
+    response = json.loads(iot.get_thing_shadow(thingName=name)["payload"].read())
     assert len(response["state"]["desired"]) == 1
     assert "state2" in response["state"]["desired"]
 
     iot.update_thing_shadow(
-        thingName=test_thing_name,
+        thingName=name,
         payload=json.dumps({"state": {"desired": {"state2": None}}}),
     )
-    response = json.loads(
-        iot.get_thing_shadow(thingName=test_thing_name)["payload"].read()
-    )
+    response = json.loads(iot.get_thing_shadow(thingName=name)["payload"].read())
     assert "desired" not in response["state"]
+
+
+@iot_aws_verified()
+@pytest.mark.aws_verified
+@pytest.mark.parametrize(
+    "desired,initial_delta,reported,delta_after_report",
+    [
+        (
+            {"desired": {"online": True}},
+            {"desired": {"online": True}, "delta": {"online": True}},
+            {"reported": {"online": False}},
+            {
+                "desired": {"online": True},
+                "reported": {"online": False},
+                "delta": {"online": True},
+            },
+        ),
+        (
+            {"desired": {"enabled": True}},
+            {"desired": {"enabled": True}, "delta": {"enabled": True}},
+            {"reported": {"online": False, "enabled": True}},
+            {
+                "desired": {"enabled": True},
+                "reported": {"online": False, "enabled": True},
+            },
+        ),
+        ({}, {}, {"reported": {"online": False}}, {"reported": {"online": False}}),
+        ({}, {}, {"reported": {"online": None}}, {}),
+        (
+            {"desired": {}},
+            {},
+            {"reported": {"online": False}},
+            {"reported": {"online": False}},
+        ),
+    ],
+)
+def test_delta_calculation(
+    desired: Dict[str, Dict[str, Optional[bool]]],
+    initial_delta: Dict[str, Dict[str, Optional[bool]]],
+    reported: Dict[str, Dict[str, Optional[bool]]],
+    delta_after_report: Dict[str, Dict[str, Optional[bool]]],
+    name: Optional[str] = None,
+) -> None:
+    client = boto3.client("iot-data", region_name="ap-northeast-1")
+    desired_payload = json.dumps({"state": desired}).encode("utf-8")
+    client.update_thing_shadow(thingName=name, payload=desired_payload)
+
+    res = client.get_thing_shadow(thingName=name)
+    payload = json.loads(res["payload"].read())
+    assert payload["state"] == initial_delta
+
+    reported_payload = json.dumps({"state": reported}).encode("utf-8")
+    client.update_thing_shadow(thingName=name, payload=reported_payload)
+
+    res = client.get_thing_shadow(thingName=name)
+    payload = json.loads(res["payload"].read())
+    assert payload["state"] == delta_after_report