fix: allow legacy ndb to read LocalStructuredProperty entities. (#344)

fix: allow legacy ndb to read LocalStructuredProperty entities

Legacy ndb is not able to read back serialized entities when using repeated LocalStructuredProperty.

Author: cguardia

packages/google-cloud-ndb/google/cloud/ndb/model.py

@@ -4174,7 +4174,7 @@ def _to_datastore(self, entity, data, prefix="", repeated=False):
         behavior to store everything in a single Datastore entity that uses
         dotted attribute names, rather than nesting entities.
         """
-        # Avoid Python 2.7 circularf import
+        # Avoid Python 2.7 circular import
         from google.cloud.ndb import context as context_module
 
         context = context_module.get_context()
@@ -4323,6 +4323,40 @@ def _prepare_for_put(self, entity):
             if value is not None:
                 value._prepare_for_put()
 
+    def _to_datastore(self, entity, data, prefix="", repeated=False):
+        """Override of :method:`Property._to_datastore`.
+
+        Although this property's entities should be stored as serialized
+        strings, when stored using old NDB they appear as unserialized
+        entities in the datastore. When serialized as strings in this class,
+        they can't be read by old NDB either. To avoid these incompatibilities,
+        we store them as entities when legacy_data is set to True, which is the
+        default behavior.
+        """
+        # Avoid Python 2.7 circular import
+        from google.cloud.ndb import context as context_module
+
+        context = context_module.get_context()
+
+        keys = super(LocalStructuredProperty, self)._to_datastore(
+            entity, data, prefix=prefix, repeated=repeated
+        )
+
+        if context.legacy_data:
+            values = self._get_user_value(entity)
+            if not self._repeated:
+                values = [values]
+            legacy_values = []
+            for value in values:
+                legacy_values.append(
+                    _entity_to_ds_entity(value, set_key=False)
+                )
+            if not self._repeated:
+                legacy_values = legacy_values[0]
+            data[self._name] = legacy_values
+
+        return keys
+
 
 class GenericProperty(Property):
     """A Property whose value can be (almost) any basic type.
@@ -5161,7 +5195,7 @@ def _put_async(self, **kwargs):
             tasklets.Future: The eventual result will be the key for the
                 entity. This is always a complete key.
         """
-        # Avoid Python 2.7 circularf import
+        # Avoid Python 2.7 circular import
         from google.cloud.ndb import context as context_module
         from google.cloud.ndb import _datastore_api
 
@@ -5378,7 +5412,7 @@ def _allocate_ids_async(
             tasklets.Future: Eventual result is ``tuple(key.Key)``: Keys for
                 the newly allocated IDs.
         """
-        # Avoid Python 2.7 circularf import
+        # Avoid Python 2.7 circular import
         from google.cloud.ndb import _datastore_api
 
         if max:

packages/google-cloud-ndb/tests/system/test_crud.py

@@ -1216,3 +1216,41 @@ class SomeKind(ndb.Model):
 
     ds_entity = ds_client.get(key._key)
     assert zlib.decompress(ds_entity["foo"]) == b"Compress this!"
+
+
+def test_insert_entity_with_repeated_local_structured_property_legacy_data(
+    client_context, dispose_of, ds_client
+):
+    """Regression test for #326
+
+    https://github.com/googleapis/python-ndb/issues/326
+    """
+
+    class OtherKind(ndb.Model):
+        one = ndb.StringProperty()
+        two = ndb.StringProperty()
+
+    class SomeKind(ndb.Model):
+        foo = ndb.IntegerProperty()
+        bar = ndb.LocalStructuredProperty(OtherKind, repeated=True)
+
+    with client_context.new(legacy_data=True).use():
+        entity = SomeKind(
+            foo=42,
+            bar=[
+                OtherKind(one="hi", two="mom"),
+                OtherKind(one="and", two="dad"),
+            ],
+        )
+        key = entity.put()
+        dispose_of(key._key)
+
+        retrieved = key.get()
+        assert retrieved.foo == 42
+        assert retrieved.bar[0].one == "hi"
+        assert retrieved.bar[0].two == "mom"
+        assert retrieved.bar[1].one == "and"
+        assert retrieved.bar[1].two == "dad"
+
+        assert isinstance(retrieved.bar[0], OtherKind)
+        assert isinstance(retrieved.bar[1], OtherKind)

packages/google-cloud-ndb/tests/unit/test_model.py

@@ -3670,6 +3670,62 @@ class SomeKind(model.Model):
         entity = SomeKind()
         SomeKind.foo._prepare_for_put(entity)  # noop
 
+    @staticmethod
+    @pytest.mark.usefixtures("in_context")
+    def test_repeated_local_structured_property():
+        class SubKind(model.Model):
+            bar = model.Property()
+
+        class SomeKind(model.Model):
+            foo = model.LocalStructuredProperty(
+                SubKind, repeated=True, indexed=False
+            )
+
+        entity = SomeKind(foo=[SubKind(bar="baz")])
+        data = {}
+        protobuf = model._entity_to_protobuf(entity.foo[0], set_key=False)
+        protobuf = protobuf.SerializePartialToString()
+        assert SomeKind.foo._to_datastore(entity, data, repeated=True) == (
+            "foo",
+        )
+        assert data == {"foo": [[protobuf]]}
+
+    @staticmethod
+    def test_legacy_repeated_local_structured_property(in_context):
+        class SubKind(model.Model):
+            bar = model.Property()
+
+        class SomeKind(model.Model):
+            foo = model.LocalStructuredProperty(
+                SubKind, repeated=True, indexed=False
+            )
+
+        with in_context.new(legacy_data=True).use():
+            entity = SomeKind(foo=[SubKind(bar="baz")])
+            data = {}
+            ds_entity = model._entity_to_ds_entity(
+                entity.foo[0], set_key=False
+            )
+            assert SomeKind.foo._to_datastore(entity, data, repeated=True) == (
+                "foo",
+            )
+            assert data == {"foo": [ds_entity]}
+
+    @staticmethod
+    def test_legacy_non_repeated_local_structured_property(in_context):
+        class SubKind(model.Model):
+            bar = model.Property()
+
+        class SomeKind(model.Model):
+            foo = model.LocalStructuredProperty(SubKind)
+
+        with in_context.new(legacy_data=True).use():
+            entity = SomeKind(foo=SubKind(bar="baz"))
+            data = {}
+            assert SomeKind.foo._to_datastore(entity, data) == ("foo",)
+            ds_entity = model._entity_to_ds_entity(entity.foo, set_key=False)
+            assert data == {"foo": ds_entity}
+
 
 class TestGenericProperty:
     @staticmethod