Adding helpers for interacting with properties in Entity protobuf.

This is because the repeated `property` message field in `Entity`
becomes a `properties` map field in `v1beta3`. This makes adding
new properties and iterating through all properties very
different, so we add a helper to ease the transition from
`v1beta2` to `v1beta3`.

Author: dhermes

gcloud/datastore/helpers.py

@@ -115,6 +115,37 @@ def _get_meaning(value_pb, is_list=False):
     return meaning
 
 
+def _new_value_pb(entity_pb, name):
+    """Add (by name) a new ``Value`` protobuf to an entity protobuf.
+
+    :type entity_pb: :class:`gcloud.datastore._generated.entity_pb2.Entity`
+    :param entity_pb: An entity protobuf to add a new property to.
+
+    :type name: string
+    :param name: The name of the new property.
+
+    :rtype: :class:`gcloud.datastore._generated.entity_pb2.Value`
+    :returns: The new ``Value`` protobuf that was added to the entity.
+    """
+    property_pb = entity_pb.property.add()
+    property_pb.name = name
+    return property_pb.value
+
+
+def _property_tuples(entity_pb):
+    """Iterator of name, ``Value`` tuples from entity properties.
+
+    :type entity_pb: :class:`gcloud.datastore._generated.entity_pb2.Entity`
+    :param entity_pb: An entity protobuf to add a new property to.
+
+    :rtype: :class:`generator`
+    :returns: An iterator that yields tuples of a name and ``Value``
+              corresponding to properties on the entity.
+    """
+    for property_pb in entity_pb.property:
+        yield property_pb.name, property_pb.value
+
+
 def entity_from_protobuf(pb):
     """Factory method for creating an entity based on a protobuf.
 
@@ -135,30 +166,29 @@ def entity_from_protobuf(pb):
     entity_meanings = {}
     exclude_from_indexes = []
 
-    for property_pb in pb.property:
-        value = _get_value_from_value_pb(property_pb.value)
-        prop_name = property_pb.name
+    for prop_name, value_pb in _property_tuples(pb):
+        value = _get_value_from_value_pb(value_pb)
         entity_props[prop_name] = value
 
         # Check if the property has an associated meaning.
-        meaning = _get_meaning(property_pb.value,
-                               is_list=isinstance(value, list))
+        is_list = isinstance(value, list)
+        meaning = _get_meaning(value_pb, is_list=is_list)
         if meaning is not None:
             entity_meanings[prop_name] = (meaning, value)
 
-        # Check if property_pb.value was indexed. Lists need to be
-        # special-cased and we require all `indexed` values in a list agree.
-        if isinstance(value, list):
+        # Check if ``value_pb`` was indexed. Lists need to be special-cased
+        # and we require all ``indexed`` values in a list agree.
+        if is_list:
             indexed_values = set(value_pb.indexed
-                                 for value_pb in property_pb.value.list_value)
+                                 for value_pb in value_pb.list_value)
             if len(indexed_values) != 1:
                 raise ValueError('For a list_value, subvalues must either all '
                                  'be indexed or all excluded from indexes.')
 
             if not indexed_values.pop():
                 exclude_from_indexes.append(prop_name)
         else:
-            if not property_pb.value.indexed:
+            if not value_pb.indexed:
                 exclude_from_indexes.append(prop_name)
 
     entity = Entity(key=key, exclude_from_indexes=exclude_from_indexes)
@@ -186,19 +216,16 @@ def entity_to_protobuf(entity):
         if value_is_list and len(value) == 0:
             continue
 
-        prop = entity_pb.property.add()
-        # Set the name of the property.
-        prop.name = name
-
+        value_pb = _new_value_pb(entity_pb, name)
         # Set the appropriate value.
-        _set_protobuf_value(prop.value, value)
+        _set_protobuf_value(value_pb, value)
 
         # Add index information to protobuf.
         if name in entity.exclude_from_indexes:
             if not value_is_list:
-                prop.value.indexed = False
+                value_pb.indexed = False
 
-            for sub_value in prop.value.list_value:
+            for sub_value in value_pb.list_value:
                 sub_value.indexed = False
 
         # Add meaning information to protobuf.
@@ -209,10 +236,10 @@ def entity_to_protobuf(entity):
             if orig_value is value:
                 # For lists, we set meaning on each sub-element.
                 if value_is_list:
-                    for sub_value_pb in prop.value.list_value:
+                    for sub_value_pb in value_pb.list_value:
                         sub_value_pb.meaning = meaning
                 else:
-                    prop.value.meaning = meaning
+                    value_pb.meaning = meaning
 
     return entity_pb
 

gcloud/datastore/test_batch.py

@@ -94,6 +94,8 @@ def test_put_entity_w_partial_key(self):
         self.assertEqual(batch._partial_key_entities, [entity])
 
     def test_put_entity_w_completed_key(self):
+        from gcloud.datastore.helpers import _property_tuples
+
         _DATASET = 'DATASET'
         _PROPERTIES = {
             'foo': 'bar',
@@ -112,17 +114,20 @@ def test_put_entity_w_completed_key(self):
 
         mutated_entity = _mutated_pb(self, batch.mutations, 'upsert')
         self.assertEqual(mutated_entity.key, key._key)
-        props = dict([(prop.name, prop.value)
-                      for prop in mutated_entity.property])
-        self.assertTrue(props['foo'].indexed)
-        self.assertFalse(props['baz'].indexed)
-        self.assertTrue(props['spam'].indexed)
-        self.assertFalse(props['spam'].list_value[0].indexed)
-        self.assertFalse(props['spam'].list_value[1].indexed)
-        self.assertFalse(props['spam'].list_value[2].indexed)
-        self.assertFalse('frotz' in props)
+
+        prop_dict = dict(_property_tuples(mutated_entity))
+        self.assertEqual(len(prop_dict), 3)
+        self.assertTrue(prop_dict['foo'].indexed)
+        self.assertFalse(prop_dict['baz'].indexed)
+        self.assertTrue(prop_dict['spam'].indexed)
+        self.assertFalse(prop_dict['spam'].list_value[0].indexed)
+        self.assertFalse(prop_dict['spam'].list_value[1].indexed)
+        self.assertFalse(prop_dict['spam'].list_value[2].indexed)
+        self.assertFalse('frotz' in prop_dict)
 
     def test_put_entity_w_completed_key_prefixed_dataset_id(self):
+        from gcloud.datastore.helpers import _property_tuples
+
         _DATASET = 'DATASET'
         _PROPERTIES = {
             'foo': 'bar',
@@ -141,15 +146,16 @@ def test_put_entity_w_completed_key_prefixed_dataset_id(self):
 
         mutated_entity = _mutated_pb(self, batch.mutations, 'upsert')
         self.assertEqual(mutated_entity.key, key._key)
-        props = dict([(prop.name, prop.value)
-                      for prop in mutated_entity.property])
-        self.assertTrue(props['foo'].indexed)
-        self.assertFalse(props['baz'].indexed)
-        self.assertTrue(props['spam'].indexed)
-        self.assertFalse(props['spam'].list_value[0].indexed)
-        self.assertFalse(props['spam'].list_value[1].indexed)
-        self.assertFalse(props['spam'].list_value[2].indexed)
-        self.assertFalse('frotz' in props)
+
+        prop_dict = dict(_property_tuples(mutated_entity))
+        self.assertEqual(len(prop_dict), 3)
+        self.assertTrue(prop_dict['foo'].indexed)
+        self.assertFalse(prop_dict['baz'].indexed)
+        self.assertTrue(prop_dict['spam'].indexed)
+        self.assertFalse(prop_dict['spam'].list_value[0].indexed)
+        self.assertFalse(prop_dict['spam'].list_value[1].indexed)
+        self.assertFalse(prop_dict['spam'].list_value[2].indexed)
+        self.assertFalse('frotz' in prop_dict)
 
     def test_delete_w_partial_key(self):
         _DATASET = 'DATASET'

gcloud/datastore/test_client.py

@@ -17,16 +17,16 @@
 
 def _make_entity_pb(dataset_id, kind, integer_id, name=None, str_val=None):
     from gcloud.datastore._generated import entity_pb2
+    from gcloud.datastore.helpers import _new_value_pb
 
     entity_pb = entity_pb2.Entity()
     entity_pb.key.partition_id.dataset_id = dataset_id
     path_element = entity_pb.key.path_element.add()
     path_element.kind = kind
     path_element.id = integer_id
     if name is not None and str_val is not None:
-        prop = entity_pb.property.add()
-        prop.name = name
-        prop.value.string_value = str_val
+        value_pb = _new_value_pb(entity_pb, name)
+        value_pb.string_value = str_val
 
     return entity_pb
 
@@ -608,6 +608,7 @@ def test_put_multi_w_single_empty_entity(self):
         self.assertRaises(ValueError, client.put_multi, Entity())
 
     def test_put_multi_no_batch_w_partial_key(self):
+        from gcloud.datastore.helpers import _property_tuples
         from gcloud.datastore.test_batch import _Entity
         from gcloud.datastore.test_batch import _Key
         from gcloud.datastore.test_batch import _KeyPB
@@ -629,12 +630,16 @@ def test_put_multi_no_batch_w_partial_key(self):
         inserts = list(mutation.insert_auto_id)
         self.assertEqual(len(inserts), 1)
         self.assertEqual(inserts[0].key, key.to_protobuf())
-        properties = list(inserts[0].property)
-        self.assertEqual(properties[0].name, 'foo')
-        self.assertEqual(properties[0].value.string_value, u'bar')
+
+        prop_list = list(_property_tuples(inserts[0]))
+        self.assertTrue(len(prop_list), 1)
+        name, value_pb = prop_list[0]
+        self.assertEqual(name, 'foo')
+        self.assertEqual(value_pb.string_value, u'bar')
         self.assertTrue(transaction_id is None)
 
     def test_put_multi_existing_batch_w_completed_key(self):
+        from gcloud.datastore.helpers import _property_tuples
         from gcloud.datastore.test_batch import _Entity
         from gcloud.datastore.test_batch import _Key
         from gcloud.datastore.test_batch import _mutated_pb
@@ -650,9 +655,12 @@ def test_put_multi_existing_batch_w_completed_key(self):
         self.assertEqual(result, None)
         mutated_entity = _mutated_pb(self, CURR_BATCH.mutations, 'upsert')
         self.assertEqual(mutated_entity.key, key.to_protobuf())
-        properties = list(mutated_entity.property)
-        self.assertEqual(properties[0].name, 'foo')
-        self.assertEqual(properties[0].value.string_value, u'bar')
+
+        prop_list = list(_property_tuples(mutated_entity))
+        self.assertTrue(len(prop_list), 1)
+        name, value_pb = prop_list[0]
+        self.assertEqual(name, 'foo')
+        self.assertEqual(value_pb.string_value, u'bar')
 
     def test_delete(self):
         _called_with = []

gcloud/datastore/test_connection.py

@@ -670,16 +670,16 @@ def test_commit_wo_transaction(self):
         from gcloud._testing import _Monkey
         from gcloud.datastore._generated import datastore_pb2
         from gcloud.datastore import connection as MUT
+        from gcloud.datastore.helpers import _new_value_pb
 
         DATASET_ID = 'DATASET'
         key_pb = self._make_key_pb(DATASET_ID)
         rsp_pb = datastore_pb2.CommitResponse()
         mutation = datastore_pb2.Mutation()
         insert = mutation.upsert.add()
         insert.key.CopyFrom(key_pb)
-        prop = insert.property.add()
-        prop.name = 'foo'
-        prop.value.string_value = u'Foo'
+        value_pb = _new_value_pb(insert, 'foo')
+        value_pb.string_value = u'Foo'
         conn = self._makeOne()
         URI = '/'.join([
             conn.api_base_url,
@@ -717,16 +717,16 @@ def test_commit_w_transaction(self):
         from gcloud._testing import _Monkey
         from gcloud.datastore._generated import datastore_pb2
         from gcloud.datastore import connection as MUT
+        from gcloud.datastore.helpers import _new_value_pb
 
         DATASET_ID = 'DATASET'
         key_pb = self._make_key_pb(DATASET_ID)
         rsp_pb = datastore_pb2.CommitResponse()
         mutation = datastore_pb2.Mutation()
         insert = mutation.upsert.add()
         insert.key.CopyFrom(key_pb)
-        prop = insert.property.add()
-        prop.name = 'foo'
-        prop.value.string_value = u'Foo'
+        value_pb = _new_value_pb(insert, 'foo')
+        value_pb.string_value = u'Foo'
         conn = self._makeOne()
         URI = '/'.join([
             conn.api_base_url,