#260: break datastore cycles via factories

gcloud/datastore/__init__.py

@@ -7,8 +7,10 @@
 ...                                 '[email protected]',
 ...                                 '/path/to/private.key')
 >>> # Then do other things...
->>> query = dataset.query().kind('EntityKind')
->>> entity = dataset.entity('EntityKind')
+>>> from gcloud.datastore.entity import Entity
+>>> from gcloud.datastore.query import Query
+>>> query = Query(dataset, 'EntityKind')
+>>> entity = Entity(dataset, 'EntityKind')
 
 The main concepts with this API are:
 
@@ -32,6 +34,14 @@
   which represents a lookup or search over the rows in the datastore.
 """
 
+# import submodules which register factories.
+import gcloud.datastore.connection
+import gcloud.datastore.dataset
+import gcloud.datastore.entity
+import gcloud.datastore.key
+import gcloud.datastore.query
+import gcloud.datastore.transaction
+
 __version__ = '0.1.2'
 
 SCOPE = ('https://www.googleapis.com/auth/datastore ',
@@ -80,7 +90,9 @@ def get_dataset(dataset_id, client_email, private_key_path):
     >>> from gcloud import datastore
     >>> dataset = datastore.get_dataset('dataset-id', email, key_path)
     >>> # Now you can do things with the dataset.
-    >>> dataset.query().kind('TestKind').fetch()
+    >>> from gcloud.datastore.query import Query
+    >>> query = Query(dataset, 'TestKind')
+    >>> query.fetch()
     [...]
 
     :type dataset_id: string

gcloud/datastore/_helpers.py

@@ -8,12 +8,53 @@
 from google.protobuf.internal.type_checkers import Int64ValueChecker
 import pytz
 
-from gcloud.datastore.entity import Entity
-from gcloud.datastore.key import Key
 
 INT_VALUE_CHECKER = Int64ValueChecker()
 
 
+class DuplicateFactory(Exception):
+    """Conflicting factory registration."""
+
+
+class InvalidFactory(Exception):
+    """Unknown factory invocation."""
+
+
+class _FactoryRegistry(object):
+    """Single registry for named factories.
+
+    This registry provides an API for modules to instantiate objects from
+    other modules without needing to import them directly, allowing us
+    to break circular import chains.
+    """
+
+    _MARKER = object()
+
+    def __init__(self):
+        self._registered = {}
+
+    def register(self, name, factory):
+        """Register a factory by name."""
+        if self._registered.get(name) not in (None, factory):
+            raise DuplicateFactory(name)
+        self._registered[name] = factory
+
+    def get(self, name, default=_MARKER):
+        """Look up a factory by name."""
+        found = self._registered.get(name, default)
+        if found is self._MARKER:
+            raise InvalidFactory(name)
+        return found
+
+    def invoke(self, name, *args, **kw):
+        """Look up and call a factory by name."""
+        return self.get(name)(*args, **kw)
+
+
+_FACTORIES = _FactoryRegistry()  # singleton
+del _FactoryRegistry
+
+
 def _get_protobuf_attribute_and_value(val):
     """Given a value, return the protobuf attribute name and proper value.
 
@@ -47,6 +88,8 @@ def _get_protobuf_attribute_and_value(val):
 
     :returns: A tuple of the attribute name and proper value type.
     """
+    key_class = _FACTORIES.get('Key')
+    entity_class = _FACTORIES.get('Entity')
 
     if isinstance(val, datetime.datetime):
         name = 'timestamp_microseconds'
@@ -58,8 +101,6 @@ def _get_protobuf_attribute_and_value(val):
         val = val.astimezone(pytz.utc)
         # Convert the datetime to a microsecond timestamp.
         value = long(calendar.timegm(val.timetuple()) * 1e6) + val.microsecond
-    elif isinstance(val, Key):
-        name, value = 'key', val.to_protobuf()
     elif isinstance(val, bool):
         name, value = 'boolean', val
     elif isinstance(val, float):
@@ -71,10 +112,12 @@ def _get_protobuf_attribute_and_value(val):
         name, value = 'string', val
     elif isinstance(val, (bytes, str)):
         name, value = 'blob', val
-    elif isinstance(val, Entity):
-        name, value = 'entity', val
     elif isinstance(val, list):
         name, value = 'list', val
+    elif key_class and isinstance(val, key_class):
+        name, value = 'key', val.to_protobuf()
+    elif entity_class and isinstance(val, entity_class):
+        name, value = 'entity', val
     else:
         raise ValueError("Unknown protobuf attr type %s" % type(val))
 
@@ -105,7 +148,7 @@ def _get_value_from_value_pb(value_pb):
         result = naive.replace(tzinfo=pytz.utc)
 
     elif value_pb.HasField('key_value'):
-        result = Key.from_protobuf(value_pb.key_value)
+        result = _FACTORIES.invoke('Key_from_protobuf', value_pb.key_value)
 
     elif value_pb.HasField('boolean_value'):
         result = value_pb.boolean_value
@@ -123,7 +166,8 @@ def _get_value_from_value_pb(value_pb):
         result = value_pb.blob_value
 
     elif value_pb.HasField('entity_value'):
-        result = Entity.from_protobuf(value_pb.entity_value)
+        result = _FACTORIES.invoke(
+            'Entity_from_protobuf', value_pb.entity_value)
 
     elif value_pb.list_value:
         result = [_get_value_from_value_pb(x) for x in value_pb.list_value]

gcloud/datastore/connection.py

@@ -210,9 +210,10 @@ def run_query(self, dataset_id, query_pb, namespace=None):
         uses this method to fetch data:
 
         >>> from gcloud import datastore
+        >>> from gcloud.datastore.query import Query
         >>> connection = datastore.get_connection(email, key_path)
         >>> dataset = connection.dataset('dataset-id')
-        >>> query = dataset.query().kind('MyKind').filter('property =', 'val')
+        >>> query = Query(dataset, 'MyKind').filter('property =', 'val')
 
         Using the `fetch`` method...
 

gcloud/datastore/dataset.py

@@ -1,5 +1,7 @@
 """Create / interact with gcloud datastore datasets."""
 
+from gcloud.datastore import _helpers
+
 
 class Dataset(object):
     """A dataset in the Cloud Datastore.
@@ -60,34 +62,6 @@ def id(self):
 
         return self._id
 
-    def query(self, *args, **kwargs):
-        """Create a query bound to this dataset.
-
-        :param args: positional arguments, passed through to the Query
-
-        :param kw: keyword arguments, passed through to the Query
-
-        :rtype: :class:`gcloud.datastore.query.Query`
-        :returns: a new Query instance, bound to this dataset.
-        """
-        # This import is here to avoid circular references.
-        from gcloud.datastore.query import Query
-        kwargs['dataset'] = self
-        return Query(*args, **kwargs)
-
-    def entity(self, kind):
-        """Create an entity bound to this dataset.
-
-        :type kind: string
-        :param kind: the "kind" of the new entity.
-
-        :rtype: :class:`gcloud.datastore.entity.Entity`
-        :returns: a new Entity instance, bound to this dataset.
-        """
-        # This import is here to avoid circular references.
-        from gcloud.datastore.entity import Entity
-        return Entity(dataset=self, kind=kind)
-
     def transaction(self, *args, **kwargs):
         """Create a transaction bound to this dataset.
 
@@ -98,10 +72,8 @@ def transaction(self, *args, **kwargs):
         :rtype: :class:`gcloud.datastore.transaction.Transaction`
         :returns: a new Transaction instance, bound to this dataset.
         """
-        # This import is here to avoid circular references.
-        from gcloud.datastore.transaction import Transaction
         kwargs['dataset'] = self
-        return Transaction(*args, **kwargs)
+        return _helpers._FACTORIES.invoke('Transaction', *args, **kwargs)
 
     def get_entity(self, key):
         """Retrieves entity from the dataset, along with its attributes.
@@ -125,15 +97,16 @@ def get_entities(self, keys):
         :rtype: list of :class:`gcloud.datastore.entity.Entity`
         :return: The requested entities.
         """
-        # This import is here to avoid circular references.
-        from gcloud.datastore.entity import Entity
-
         entity_pbs = self.connection().lookup(
             dataset_id=self.id(),
             key_pbs=[k.to_protobuf() for k in keys]
         )
 
         entities = []
         for entity_pb in entity_pbs:
-            entities.append(Entity.from_protobuf(entity_pb, dataset=self))
+            entities.append(_helpers._FACTORIES.invoke(
+                'Entity_from_protobuf', entity_pb, dataset=self))
         return entities
+
+
+_helpers._FACTORIES.register('Dataset', Dataset)

gcloud/datastore/demo/demo.py

@@ -6,10 +6,13 @@
 
 # Let's start by importing the demo module and getting a dataset:
 from gcloud.datastore import demo
+from gcloud.datastore.entity import Entity
+from gcloud.datastore.query import Query
+
 dataset = demo.get_dataset()
 
 # Let's create a new entity of type "Thing" and name it 'Toy':
-toy = dataset.entity('Thing')
+toy = Entity(dataset, 'Thing')
 toy.update({'name': 'Toy'})
 
 # Now let's save it to our datastore:
@@ -26,7 +29,7 @@
 
 # Now let's try a more advanced query.
 # We'll start by look at all Thing entities:
-query = dataset.query().kind('Thing')
+query = Query(dataset, 'Thing')
 
 # Let's look at the first two.
 print query.limit(2).fetch()
@@ -42,13 +45,13 @@
 # (Check the official docs for explanations of what's happening here.)
 with dataset.transaction():
     print 'Creating and savng an entity...'
-    thing = dataset.entity('Thing')
+    thing = Entity(dataset, 'Thing')
     thing.key(thing.key().name('foo'))
     thing['age'] = 10
     thing.save()
 
     print 'Creating and saving another entity...'
-    thing2 = dataset.entity('Thing')
+    thing2 = Entity(dataset, 'Thing')
     thing2.key(thing2.key().name('bar'))
     thing2['age'] = 15
     thing2.save()
@@ -60,7 +63,7 @@
 
 # To rollback a transaction, just call .rollback()
 with dataset.transaction() as t:
-    thing = dataset.entity('Thing')
+    thing = Entity(dataset, 'Thing')
     thing.key(thing.key().name('another'))
     thing.save()
     t.rollback()
@@ -72,7 +75,7 @@
 # Remember, a key won't be complete until the transaction is commited.
 # That is, while inside the transaction block, thing.key() will be incomplete.
 with dataset.transaction():
-    thing = dataset.entity('Thing')
+    thing = Entity(dataset, 'Thing')
     thing.save()
     print thing.key()  # This will be partial
 

gcloud/datastore/entity.py

@@ -16,7 +16,7 @@
 """
 
 from gcloud.datastore import datastore_v1_pb2 as datastore_pb
-from gcloud.datastore.key import Key
+from gcloud.datastore import _helpers
 
 
 class NoKey(RuntimeError):
@@ -39,15 +39,10 @@ class Entity(dict):
     This means you could take an existing entity and change the key
     to duplicate the object.
 
-    This can be used on its own, however it is likely easier to use
-    the shortcut methods provided by :class:`gcloud.datastore.dataset.Dataset`
+    Entities can be constructed directly, or obtained via
+    the "lookup" methods provided by :class:`gcloud.datastore.dataset.Dataset`
     such as:
 
-    - :func:`gcloud.datastore.dataset.Dataset.entity` to create a new entity.
-
-      >>> dataset.entity('MyEntityKind')
-      <Entity[{'kind': 'MyEntityKind'}] {}>
-
     - :func:`gcloud.datastore.dataset.Dataset.get_entity`
       to retrieve an existing entity.
 
@@ -74,7 +69,7 @@ def __init__(self, dataset=None, kind=None):
         super(Entity, self).__init__()
         self._dataset = dataset
         if kind:
-            self._key = Key().kind(kind)
+            self._key = _helpers._FACTORIES.invoke('Key').kind(kind)
         else:
             self._key = None
 
@@ -159,11 +154,7 @@ def from_protobuf(cls, pb, dataset=None):
         :returns: The :class:`Entity` derived from the
                   :class:`gcloud.datastore.datastore_v1_pb2.Entity`.
         """
-
-        # This is here to avoid circular imports.
-        from gcloud.datastore import _helpers
-
-        key = Key.from_protobuf(pb.key)
+        key = _helpers._FACTORIES.invoke('Key_from_protobuf', pb.key)
         entity = cls.from_key(key, dataset)
 
         for property_pb in pb.property:
@@ -248,7 +239,8 @@ def save(self):
             transaction.add_auto_id_entity(self)
 
         if isinstance(key_pb, datastore_pb.Key):
-            updated_key = Key.from_protobuf(key_pb)
+            updated_key = _helpers._FACTORIES.invoke(
+                'Key_from_protobuf', key_pb)
             # Update the path (which may have been altered).
             self._key = key.path(updated_key.path())
 
@@ -275,3 +267,7 @@ def __repr__(self):
                                       super(Entity, self).__repr__())
         else:
             return '<Entity %s>' % (super(Entity, self).__repr__())
+
+
+_helpers._FACTORIES.register('Entity', Entity)
+_helpers._FACTORIES.register('Entity_from_protobuf', Entity.from_protobuf)

gcloud/datastore/key.py

@@ -4,6 +4,7 @@
 from itertools import izip
 
 from gcloud.datastore import datastore_v1_pb2 as datastore_pb
+from gcloud.datastore import _helpers
 
 
 class Key(object):
@@ -251,3 +252,8 @@ def parent(self):
 
     def __repr__(self):
         return '<Key%s>' % self.path()
+
+
+_helpers._FACTORIES.register('Key', Key)
+_helpers._FACTORIES.register('Key_from_protobuf', Key.from_protobuf)
+_helpers._FACTORIES.register('Key_from_path', Key.from_path)